Repository: mitdbg/palimpzest Branch: main Commit: 807ed301c4d2 Files: 250 Total size: 3.6 MB Directory structure: gitextract_v67onyjf/ ├── .github/ │ └── workflows/ │ ├── ci.yaml │ ├── docs.yaml │ ├── package.yaml │ └── test-docs.yaml ├── .gitignore ├── LICENSE ├── README.md ├── abacus-research/ │ ├── README.md │ ├── README_CUAD_LOCAL.md │ ├── biodex-ablation.py │ ├── biodex-demo.py │ ├── biodex-max-quality-at-cost.py │ ├── biodex-min-at-fixed-quality.py │ ├── biodex-pareto-cascades.py │ ├── biodex-priors-cascades.json │ ├── biodex-priors.json │ ├── biodex-revision-priors-maxquality.json │ ├── biodex-revision-priors-mincost.json │ ├── cheap-priors-cascades.json │ ├── cheap-priors.json │ ├── cuad-demo.py │ ├── cuad-max-quality-at-cost.py │ ├── cuad-priors.json │ ├── cuad_data_loader.py │ ├── download_embeddings_and_mmqa.sh │ ├── helper-scripts/ │ │ ├── biodex-gen-index.py │ │ ├── generate-prior-stats-biodex-first-convert.py │ │ ├── generate-prior-stats-biodex.py │ │ ├── generate-prior-stats-cuad.py │ │ ├── mmqa-baseline.py │ │ ├── mmqa-gen-image-index.py │ │ ├── mmqa-gen-image-title-index.py │ │ ├── mmqa-gen-table-index.py │ │ └── mmqa-gen-text-index.py │ ├── mmqa-complex-demo.py │ ├── mmqa-demo.py │ ├── run_ablation_study.sh │ ├── run_biodex.sh │ ├── run_biodex_cascades.sh │ ├── run_biodex_cost_threshold.sh │ ├── run_biodex_min_cost_latency.sh │ ├── run_biodex_priors.sh │ ├── run_biodex_priors_constrained.sh │ ├── run_cuad.sh │ ├── run_cuad_cost_threshold.sh │ ├── run_cuad_min_cost_latency.sh │ ├── run_cuad_priors.sh │ ├── run_cuad_priors_constrained.sh │ ├── run_mmqa.sh │ ├── run_mmqa_complex.sh │ ├── run_mmqa_complex_min_cost_latency.sh │ ├── run_mmqa_min_cost_latency.sh │ ├── score_biodex.py │ ├── score_cuad.py │ ├── score_mmqa.py │ ├── score_mmqa_complex.py │ └── setup_cuad_data.py ├── demos/ │ ├── audio-demo.py │ ├── caching-demo.py │ ├── demo_core.py │ ├── enron-demo.py │ ├── image-demo.py │ ├── join-data/ │ │ └── animal-texts/ │ │ ├── animal1.txt │ │ ├── animal2.txt │ │ ├── animal3.txt │ │ ├── animal4.txt │ │ ├── animal5.txt │ │ └── animal6.txt │ ├── join-demo.py │ ├── paper-demo.py │ ├── real-estate-demo.py │ ├── simple-demo.py │ └── vllm-demo.py ├── evals/ │ └── quest/ │ └── eval.py ├── pyproject.toml ├── quickstart.ipynb ├── ruff.toml ├── scripts/ │ ├── capture_litellm_stats.py │ ├── capture_provider_stats.py │ ├── generate_test_messages.py │ └── update_model_info.py ├── src/ │ └── palimpzest/ │ ├── __init__.py │ ├── agents/ │ │ ├── __init__.py │ │ ├── compute_agents.py │ │ └── search_agents.py │ ├── constants.py │ ├── core/ │ │ ├── __init__.py │ │ ├── data/ │ │ │ ├── __init__.py │ │ │ ├── context.py │ │ │ ├── context_manager.py │ │ │ ├── dataset.py │ │ │ ├── index_dataset.py │ │ │ └── iter_dataset.py │ │ ├── elements/ │ │ │ ├── __init__.py │ │ │ ├── filters.py │ │ │ ├── groupbysig.py │ │ │ └── records.py │ │ ├── lib/ │ │ │ ├── __init__.py │ │ │ └── schemas.py │ │ └── models.py │ ├── policy.py │ ├── prompts/ │ │ ├── __init__.py │ │ ├── agent_prompts.py │ │ ├── aggregate_prompts.py │ │ ├── context_search.py │ │ ├── convert_prompts.py │ │ ├── critique_and_refine_prompts.py │ │ ├── filter_prompts.py │ │ ├── join_prompts.py │ │ ├── moa_aggregator_prompts.py │ │ ├── moa_proposer_prompts.py │ │ ├── prompt_factory.py │ │ ├── prompt_manager.py │ │ ├── split_merge_prompts.py │ │ ├── split_proposer_prompts.py │ │ ├── utils.py │ │ └── validator.py │ ├── query/ │ │ ├── __init__.py │ │ ├── execution/ │ │ │ ├── __init__.py │ │ │ ├── all_sample_execution_strategy.py │ │ │ ├── execution_strategy.py │ │ │ ├── execution_strategy_type.py │ │ │ ├── mab_execution_strategy.py │ │ │ ├── parallel_execution_strategy.py │ │ │ └── single_threaded_execution_strategy.py │ │ ├── generators/ │ │ │ ├── __init__.py │ │ │ ├── gemini_client.py │ │ │ └── generators.py │ │ ├── operators/ │ │ │ ├── __init__.py │ │ │ ├── aggregate.py │ │ │ ├── compute.py │ │ │ ├── convert.py │ │ │ ├── critique_and_refine.py │ │ │ ├── distinct.py │ │ │ ├── filter.py │ │ │ ├── join.py │ │ │ ├── limit.py │ │ │ ├── logical.py │ │ │ ├── mixture_of_agents.py │ │ │ ├── physical.py │ │ │ ├── project.py │ │ │ ├── rag.py │ │ │ ├── scan.py │ │ │ ├── search.py │ │ │ ├── split.py │ │ │ └── topk.py │ │ ├── optimizer/ │ │ │ ├── __init__.py │ │ │ ├── cost_model.py │ │ │ ├── optimizer.py │ │ │ ├── optimizer_strategy.py │ │ │ ├── optimizer_strategy_type.py │ │ │ ├── plan.py │ │ │ ├── primitives.py │ │ │ ├── rules.py │ │ │ └── tasks.py │ │ └── processor/ │ │ ├── __init__.py │ │ ├── config.py │ │ ├── query_processor.py │ │ └── query_processor_factory.py │ ├── schemabuilder/ │ │ ├── __init__.py │ │ └── schema_builder.py │ ├── tools/ │ │ ├── README.md │ │ ├── __init__.py │ │ ├── allenpdf.py │ │ ├── pdfparser.py │ │ └── skema_tools.py │ ├── utils/ │ │ ├── __init__.py │ │ ├── env_helpers.py │ │ ├── hash_helpers.py │ │ ├── model_helpers.py │ │ ├── model_info_helpers.py │ │ ├── progress.py │ │ ├── pz_models_information.json │ │ └── udfs.py │ └── validator/ │ ├── __init__.py │ └── validator.py ├── testdata/ │ ├── README.md │ ├── download-testdata.sh │ ├── enron-eval-medium-labels.json │ └── target_matching.csv ├── tests/ │ └── pytest/ │ ├── README.md │ ├── conftest.py │ ├── data/ │ │ ├── email_schema.json │ │ ├── email_schema.yml │ │ ├── synapse_schema.csv │ │ └── synapse_schema.jsonld │ ├── fixtures/ │ │ ├── champion_outputs.py │ │ ├── datasets.py │ │ ├── execution_data.py │ │ ├── expected_physical_plans.py │ │ ├── expected_qualities.py │ │ ├── expected_records.py │ │ ├── models.py │ │ ├── operator_to_stats.py │ │ ├── physical_plans.py │ │ ├── schemas.py │ │ ├── side_effects.py │ │ └── workloads.py │ ├── test_aggregate.py │ ├── test_convert.py │ ├── test_dataset.py │ ├── test_distinct.py │ ├── test_dynamic_models.py │ ├── test_dynamicschema.py │ ├── test_execution.py │ ├── test_filter.py │ ├── test_generator.py │ ├── test_iter_dataset.py │ ├── test_join.py │ ├── test_map.py │ ├── test_optimizer.py │ ├── test_physical.py │ ├── test_records.py │ ├── test_rules.py │ ├── test_scan.py │ └── test_schemas.py └── website/ ├── .gitignore ├── README.md ├── blog/ │ ├── 2024-06-01-palimpzest/ │ │ ├── bibtex.js │ │ └── index.md │ ├── authors.yml │ └── tags.yml ├── docs/ │ ├── api/ │ │ └── overview.mdx │ ├── getting-started/ │ │ ├── installation.mdx │ │ ├── next-steps.mdx │ │ └── quickstart.mdx │ ├── intro.mdx │ └── user-guide/ │ ├── dataset.mdx │ ├── operators/ │ │ ├── overview.mdx │ │ ├── relational.mdx │ │ ├── sem_agg.mdx │ │ ├── sem_filter.mdx │ │ ├── sem_join.mdx │ │ ├── sem_map.mdx │ │ └── sem_topk.mdx │ ├── optimization.mdx │ └── overview.mdx ├── docusaurus.config.ts ├── package.json ├── sidebars.ts ├── src/ │ ├── components/ │ │ ├── HomepageFeatures/ │ │ │ ├── index.tsx │ │ │ └── styles.module.css │ │ └── ResearchPage/ │ │ └── admonitions.tsx │ ├── css/ │ │ └── custom.css │ └── pages/ │ ├── index.module.css │ ├── index.tsx │ ├── palimpchat.mdx │ └── research.mdx ├── static/ │ └── .nojekyll └── tsconfig.json ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/ci.yaml ================================================ name: PZ Merge Checks on: pull_request: branches: - main jobs: test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.12' - name: Install dependencies run: | pip install --upgrade pip pip install . - name: Download and register testdata run: | pushd testdata wget -nc https://people.csail.mit.edu/gerarvit/PalimpzestData/enron-eval-tiny.tar.gz wget -nc https://people.csail.mit.edu/gerarvit/PalimpzestData/real-estate-eval-tiny.tar.gz tar -xzf enron-eval-tiny.tar.gz tar -xzf real-estate-eval-tiny.tar.gz rm *.tar.gz popd - name: Test with pytest env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} run: | export CI=true export NO_GEMINI=true pip install pytest pytest -v tests/pytest lint-and-format: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.x' - name: Install the code linting and formatting tool Ruff run: pip install "ruff>=0.9.0" - name: check version run: ruff --version - name: Lint code with Ruff run: ruff check --output-format=github --target-version=py38 - name: Check code formatting with Ruff run: ruff check --no-fix . --target-version=py38 continue-on-error: true check-version-bump: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Check Version Increased run: | git fetch --prune --unshallow git checkout ${{ github.event.pull_request.base.sha }} VERSION=`cat pyproject.toml | grep '^version' | sed -E 's/version.*=.*\"(.*)".*/\1/'` echo "Current version is $VERSION" git checkout ${{ github.event.pull_request.head.sha }} VERSION_PR=`cat pyproject.toml | grep '^version' | sed -E 's/version.*=.*\"(.*)".*/\1/'` echo "Version in PR is $VERSION_PR" if [ "$VERSION" = "$VERSION_PR" ]; then echo "Error: Version has not been bumped" exit 1 fi ================================================ FILE: .github/workflows/docs.yaml ================================================ name: Deploy Docs to GitHub Pages on: push: branches: - main permissions: contents: write jobs: build: name: Build Docusaurus runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: actions/setup-node@v4 with: node-version: 20 cache: npm cache-dependency-path: website/package-lock.json - name: Install dependencies run: | cd website npm ci - name: Build website run: | cd website npm run build echo "palimpzest.org" > build/CNAME - name: Upload Build Artifact uses: actions/upload-pages-artifact@v3 with: path: website/build deploy: name: Deploy to GitHub Pages needs: build # Grant GITHUB_TOKEN the permissions required to make a Pages deployment permissions: pages: write # to deploy to Pages id-token: write # to verify the deployment originates from an appropriate source # Deploy to the github-pages environment environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest steps: - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v4 ================================================ FILE: .github/workflows/package.yaml ================================================ name: package on: push: branches: - main pull_request: branches: - main jobs: build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.x' - name: Build Package run: | pip install --upgrade pip build python3 -m build - name: Store the distribution packages uses: actions/upload-artifact@v4 with: name: python-package-distributions path: dist/ publish: runs-on: ubuntu-latest name: Publish Package if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} needs: - build environment: name: pypi url: https://pypi.org/p/palimpzest permissions: id-token: write steps: - name: Download all the dists uses: actions/download-artifact@v4 with: name: python-package-distributions path: dist/ - name: Publish distribution to PyPI uses: pypa/gh-action-pypi-publish@release/v1 github-release: name: >- Sign distribution w/Sigstore and upload to GitHub Release needs: - publish runs-on: ubuntu-latest permissions: contents: write id-token: write steps: - name: Download all the dists uses: actions/download-artifact@v4 with: name: python-package-distributions path: dist/ - name: Sign the dists with Sigstore uses: sigstore/gh-action-sigstore-python@v3.0.0 with: inputs: >- ./dist/*.tar.gz ./dist/*.whl - name: Create GitHub Release env: GITHUB_TOKEN: ${{ github.token }} run: | PKG_VERSION=`ls dist/ | head -n 1 | sed -E 's/.*palimpzest-([0-9]+\.[0-9]+\.[0-9]+)-.*/\1/'` gh release create "$PKG_VERSION" --repo "$GITHUB_REPOSITORY" --notes "" - name: Upload artifact signatures to GitHub Release env: GITHUB_TOKEN: ${{ github.token }} # Upload to GitHub Release using the `gh` CLI. # `dist/` contains the built packages, and the # sigstore-produced signatures and certificates. run: | PKG_VERSION=`ls dist/ | head -n 1 | sed -E 's/.*palimpzest-([0-9]+\.[0-9]+\.[0-9]+)-.*/\1/'` gh release upload "$PKG_VERSION" dist/** --repo "$GITHUB_REPOSITORY" ================================================ FILE: .github/workflows/test-docs.yaml ================================================ name: Test Building Docs on: pull_request: branches: - main jobs: test-deploy: name: Test deployment runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: actions/setup-node@v4 with: node-version: 20 cache: npm cache-dependency-path: website/package-lock.json - name: Install dependencies run: | cd website npm ci - name: Test build website run: | cd website npm run build ================================================ FILE: .gitignore ================================================ docs/site/ *.zip .cache/ .env build/* docs/build/* docs/source/generated/* dist/* .vscode/* .idea/* .chroma .chroma-biodex .chroma-mmqa .ragatouille plots/ paper-imgs/ # testdata folders and archive files testdata/enron-tiny.csv testdata/*/ testdata/*.tar.gz tests/pytest/data/generator_messages/ scripts/provider_stats/ scripts/litellm_stats/ # python artifacts *.egg-info **/__pycache__/ # other .DS_Store # logs *.log # virtual environment(s) venv/ uv.lock # tmp testdata/maildir/ testdata/real-estate-eval-100.tar # jupyter .ipynb_checkpoints/ # evaluation old-eval-results/ eval-results/ testdata/enron-eval/*.txt # your zed using open source contributor who only installs in a virtual environment .venv .zed pyrightconfig.json myenv/ pz-env/ # abacus-research data abacus-research/cuad-data/* abacus-research/opt-profiling-data/* abacus-research/parse-answer-errors/* # stats scripts/litellm_stats/ scripts/provider_stats/ tests/pytest/data/generator_messages/ ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2024 MIT Data Systems Group Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ ![pz-banner](https://palimpzest-workloads.s3.us-east-1.amazonaws.com/palimpzest-cropped.png) # Palimpzest (PZ) [![Discord](https://img.shields.io/discord/1245561987480420445?logo=discord)](https://discord.gg/dN85JJ6jaH) [![Docs](https://img.shields.io/badge/Read_the_Docs-purple?logo=readthedocs)](https://palimpzest.org/) [![Colab Demo](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1Fm8I4yL1az395MsFkQbEIZSmUZs0oGvZ?usp=sharing) [![PyPI](https://img.shields.io/pypi/v/palimpzest)](https://pypi.org/project/palimpzest/) [![PyPI - Monthly Downloads](https://img.shields.io/pypi/dm/palimpzest?color=teal)](https://pypi.org/project/palimpzest/) ## 📚 Learn How to Use PZ Our [full documentation](https://palimpzest.org) is the definitive resource for learning how to use PZ. It contains all of the installation and quickstart materials on this page, as well as user guides, full API documentation (coming soon), and much more. ## 🚀 Getting started You can find a stable version of the PZ package on PyPI [here](https://pypi.org/project/palimpzest/). To install the package, run: ```bash $ pip install palimpzest ``` You can also install PZ with [uv](https://docs.astral.sh/uv/) for a faster installation: ```bash $ uv pip install palimpzest ``` Alternatively, to install the latest version of the package from this repository, you can clone this repository and run the following commands: ```bash $ git clone git@github.com:mitdbg/palimpzest.git $ cd palimpzest $ pip install . ``` ## 🙋🏽 Join the PZ Community We are actively hacking on PZ and would love to have you join our community [![Discord](https://img.shields.io/discord/1245561987480420445?logo=discord)](https://discord.gg/dN85JJ6jaH) [Our Discord server](https://discord.gg/dN85JJ6jaH) is the best place to: - Get help with your PZ program(s) - Give feedback to the maintainers - Discuss the future direction(s) of the project - Discuss anything related to data processing with LLMs! We are eager to learn more about your workloads and use cases, and will take them into consideration in planning our future roadmap. ### 📓 Citation If you would like to cite our original paper on Palimpzest, please use the following citation: ``` @inproceedings{palimpzestCIDR, title={Palimpzest: Optimizing AI-Powered Analytics with Declarative Query Processing}, author={Liu, Chunwei and Russo, Matthew and Cafarella, Michael and Cao, Lei and Chen, Peter Baile and Chen, Zui and Franklin, Michael and Kraska, Tim and Madden, Samuel and Shahout, Rana and Vitagliano, Gerardo}, booktitle = {Proceedings of the {{Conference}} on {{Innovative Database Research}} ({{CIDR}})}, date = 2025, } ``` If you would like to cite our paper on Palimpzest's optimizer Abacus, please use the following citation: ``` @misc{russo2025abacuscostbasedoptimizersemantic, title={Abacus: A Cost-Based Optimizer for Semantic Operator Systems}, author={Matthew Russo and Sivaprasad Sudhir and Gerardo Vitagliano and Chunwei Liu and Tim Kraska and Samuel Madden and Michael Cafarella}, year={2025}, eprint={2505.14661}, archivePrefix={arXiv}, primaryClass={cs.DB}, url={https://arxiv.org/abs/2505.14661}, } ``` ================================================ FILE: abacus-research/README.md ================================================ ## Chroma Embeddings and MMQA files You can download the chroma embeddings we computed for MMQA and BioDEX by executing the following: ```sh $ ./download_embeddings_and_mmqa.sh ``` This folder also contains questions for the different splits of MMQA -- of which we only use `MMQA_dev.jsonl` for scoring PZ's output. If you need the full MMQA dataset for any reason (e.g. to visualize at which images are being retrieved by a pipeline), you can find it here: https://github.com/allenai/multimodalqa/tree/master. ## Table 2 The following scripts create the data for Abacus in Table 2 in our Abacus paper. - `run_biodex.sh` - `run_cuad.sh` - `run_mmqa_complex.sh` ## Table 3 The following scripts create the data for Abacus in Table 3 in our Abacus paper. - `run_biodex_min_cost_latency.sh` - `run_cuad_min_cost_latency.sh` - `run_mmqa_complex_min_cost_latency.sh` ## Figure 6 The following scripts create the data for Figure 6 in our Abacus paper. - `run_biodex_priors.sh` - `run_biodex_priors_constrained.sh` - `run_cuad_priors.sh` - `run_cuad_priors_constrained.sh` ## Figure 7 The `run_biodex_cost_threshold.sh` and `run_cuad_cost_threshold.sh` scripts create the data for Figure 6 in our Abacus paper. ## Figure 8 The `run_ablation_study.sh` script creates the data for Figure 8 in our Abacus paper. ================================================ FILE: abacus-research/README_CUAD_LOCAL.md ================================================ # CUAD Local Data Setup and Usage ## Setup Since HuggingFace datasets no longer supports loading scripts, we've created a local data loading solution. ### 1. Download CUAD Data First, run the setup script to download CUAD data to a local directory: ```bash python setup_cuad_data.py ``` This will: - Create a `cuad-data/` directory - Download the CUAD dataset files (train and test JSON files) - Download the original dataset script from HuggingFace for reference ### 2. Updated Scripts The following scripts have been updated to use local data via `cuad_data_loader.py`: - **cuad-demo.py** - **cuad-max-quality-at-cost.py** ### 3. Running the Scripts #### Basic CUAD Demo ```bash # Make sure OPENAI_API_KEY is set in .env or environment source ../.env && export OPENAI_API_KEY # Run from abacus-research directory seed=0 exp_name="cuad-final-mab-k6-j4-budget50-seed${seed}" python cuad-demo.py --k 6 --j 4 --sample-budget 50 --seed $seed --exp-name $exp_name --gpt4-mini-only ``` #### Max Quality at Cost ```bash python cuad-max-quality-at-cost.py --constrained --gpt4-mini-only ``` ================================================ FILE: abacus-research/biodex-ablation.py ================================================ import argparse import json import os import time import chromadb import datasets from chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction import palimpzest as pz from palimpzest.constants import Model biodex_entry_cols = [ {"name": "pmid", "type": str, "desc": "The PubMed ID of the medical paper"}, {"name": "title", "type": str, "desc": "The title of the medical paper"}, {"name": "abstract", "type": str, "desc": "The abstract of the medical paper"}, {"name": "fulltext", "type": str, "desc": "The full text of the medical paper, which contains information relevant for creating a drug safety report."}, ] biodex_reactions_cols = [ {"name": "reactions", "type": list[str], "desc": "The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible."}, ] biodex_reaction_labels_cols = [ {"name": "reaction_labels", "type": list[str], "desc": "Official terms for medical conditions listed in `reactions`"}, ] biodex_ranked_reactions_labels_cols = [ {"name": "ranked_reaction_labels", "type": list[str], "desc": "The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs."}, ] class BiodexValidator(pz.Validator): def __init__( self, rp_at_k: int = 5, num_samples: int = 5, shuffle: bool = False, seed: int = 42, ): super().__init__() # read dataset and prepare entries dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split="train").to_pandas() if shuffle: dataset = dataset.sample(n=num_samples, random_state=seed).to_dict(orient="records") else: dataset = dataset.to_dict(orient="records")[:num_samples] # compute mapping from pmid --> label (i.e. reactions list) self.pmid_to_label = self._compute_pmid_to_label(dataset) # store rp_at_k for computing rank-precision at k metric self.k = rp_at_k def _compute_pmid_to_label(self, dataset: list[dict]) -> dict: """Compute the label for a BioDEX report given its entry in the dataset.""" pmid_to_label = {} for entry in dataset: pmid = str(entry["pmid"]) reactions_lst = [ reaction.strip().lower().replace("'", "").replace("^", "") for reaction in entry["reactions"].split(",") ] pmid_to_label[pmid] = reactions_lst return pmid_to_label def rank_precision_at_k(self, preds: list | None, targets: list): if preds is None: return 0.0 try: # lower-case each list preds = [pred.strip().lower().replace("'", "").replace("^", "") for pred in preds] targets = set([target.strip().lower().replace("'", "").replace("^", "") for target in targets]) # compute rank-precision at k rn = len(targets) denom = min(self.k, rn) total = 0.0 for i in range(self.k): total += preds[i] in targets if i < len(preds) else 0.0 return total / denom except Exception: os.makedirs("rp@k-errors", exist_ok=True) ts = time.time() with open(f"rp@k-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def term_recall(self, preds: list | None, targets: list): if preds is None: return 0.0 try: # normalize terms in each list pred_terms = set([ term.strip() for pred in preds for term in pred.lower().replace("'", "").replace("^", "").split(" ") ]) target_terms = ([ term.strip() for target in targets for term in target.lower().replace("'", "").replace("^", "").split(" ") ]) # compute term recall and return intersect = pred_terms.intersection(target_terms) term_recall = len(intersect) / len(target_terms) return term_recall except Exception: os.makedirs("term-recall-eval-errors", exist_ok=True) ts = time.time() with open(f"term-recall-eval-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: field_name = fields[0] if field_name == "reactions": preds = output.get(field_name) targets = self.pmid_to_label[str(input_record["pmid"])] return self.term_recall(preds, targets) elif field_name == "ranked_reaction_labels": preds = output.get(field_name) targets = self.pmid_to_label[str(input_record["pmid"])] return self.rank_precision_at_k(preds, targets) else: raise NotImplementedError(f"Validator.map_score_fn not implemented for field {field_name}.") def topk_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: field_name = fields[0] if field_name == "reaction_labels": preds = output.get(field_name) targets = self.pmid_to_label[input_record["pmid"]] return self.term_recall(preds, targets) else: raise NotImplementedError(f"Validator.topk_score_fn not implemented for field {field_name}.") class BiodexDataset(pz.IterDataset): def __init__( self, rp_at_k: int = 5, num_samples: int = 5, split: str = "test", shuffle: bool = False, seed: int = 42, ): super().__init__(id="biodex", schema=biodex_entry_cols) self.dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split=split).to_pandas() if shuffle: self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient="records") else: self.dataset = self.dataset.to_dict(orient="records")[:num_samples] self.rp_at_k = rp_at_k self.num_samples = num_samples self.shuffle = shuffle self.seed = seed self.split = split def __len__(self): return len(self.dataset) def __getitem__(self, idx: int): # get entry entry = self.dataset[idx] # get input fields pmid = entry["pmid"] title = entry["title"] abstract = entry["abstract"] fulltext = entry["fulltext"] # create item with fields item = {"pmid": pmid, "title": title, "abstract": abstract, "fulltext": fulltext} return item if __name__ == "__main__": # parse arguments parser = argparse.ArgumentParser(description="Run a simple demo") parser.add_argument( "--optimizer-strategy", default="pareto", type=str, help="The optimizer strategy to use. One of pareto or greedy", ) parser.add_argument( "--seed", default=42, type=int, help="Seed used to initialize RNG for MAB sampling algorithm", ) parser.add_argument( "--k", default=10, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--j", default=3, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--sample-budget", default=100, type=int, help="Total sample budget in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--exp-name", default=None, type=str, help="The experiment name.", ) parser.add_argument( "--policy", default=None, type=str, help="The policy (one of 'mincost' or 'maxquality').", ) parser.add_argument( "--priors-file", default=None, type=str, help="A file with a dictionary mapping physical operator ids to prior belief on their performance", ) args = parser.parse_args() # create directory for profiling data os.makedirs("ablation-data", exist_ok=True) seed = args.seed k = args.k j = args.j sample_budget = args.sample_budget optimizer_strategy = args.optimizer_strategy exp_name = args.exp_name priors = None if args.priors_file is not None and os.path.exists(args.priors_file): with open(args.priors_file) as f: priors = json.load(f) # set the optimization policy; constraint set to 80% of mean quality from unconstrained plans (Table 2) policy = ( pz.MinCostAtFixedQuality(min_quality=0.8 * 0.261) if args.policy == "mincost" else pz.MaxQualityAtFixedCost(max_cost=0.5 * 0.7) ) print(f"USING POLICY: {policy}") if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") # create validator validator = BiodexValidator( rp_at_k=5, num_samples=20, shuffle=True, seed=seed, ) # create train dataset for validator train_dataset = BiodexDataset( split="train", num_samples=20, shuffle=True, seed=seed, ) train_dataset = {train_dataset.id: train_dataset} # load index [text-embedding-3-small] chroma_client = chromadb.PersistentClient(".chroma-biodex") openai_ef = OpenAIEmbeddingFunction( api_key=os.environ["OPENAI_API_KEY"], model_name="text-embedding-3-small", ) index = chroma_client.get_collection("biodex-reaction-terms", embedding_function=openai_ef) def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]: # execute query with embeddings results = index.query(query, n_results=5) # get list of result terms with their cosine similarity scores final_results = [] for query_docs, query_distances in zip(results["documents"], results["distances"]): for doc, dist in zip(query_docs, query_distances): cosine_similarity = 1 - dist final_results.append({"content": doc, "similarity": cosine_similarity}) # sort the results by similarity score sorted_results = sorted(final_results, key=lambda result: result["similarity"], reverse=True) # remove duplicates sorted_results_set = set() final_sorted_results = [] for result in sorted_results: if result["content"] not in sorted_results_set: sorted_results_set.add(result["content"]) final_sorted_results.append(result["content"]) # return the top-k similar results and generation stats return {"reaction_labels": final_sorted_results[:k]} # construct plan plan = BiodexDataset(split="test", num_samples=250, shuffle=True, seed=seed) plan = plan.sem_map(biodex_reactions_cols) plan = plan.sem_topk( index=index, search_func=search_func, search_attr="reactions", output_attrs=biodex_reaction_labels_cols, ) plan = plan.sem_map(biodex_ranked_reactions_labels_cols, depends_on=["title", "abstract", "fulltext", "reaction_labels"]) # set models models = [ Model.GPT_4o, Model.GPT_4o_MINI, Model.LLAMA3_1_8B, Model.LLAMA3_3_70B, # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments` # Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B, ] # execute pz plan config = pz.QueryProcessorConfig( policy=policy, optimizer_strategy=optimizer_strategy, execution_strategy="parallel", use_final_op_quality=True, max_workers=64, available_models=models, allow_bonded_query=True, allow_critic=True, allow_mixtures=True, allow_rag_reduction=True, progress=True, k=k, j=j, sample_budget=sample_budget, # sample_cost_budget=0.10, seed=seed, exp_name=exp_name, priors=priors, dont_use_priors=(priors is None), ) data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator) print(data_record_collection.to_df()) data_record_collection.to_df().to_csv(f"ablation-data/{exp_name}-output.csv", index=False) # create filepaths for records and stats records_path = f"ablation-data/{exp_name}-records.json" stats_path = f"ablation-data/{exp_name}-profiling.json" # save record outputs record_jsons = [] for record in data_record_collection: record_dict = record.to_dict() record_dict = { k: v for k, v in record_dict.items() if k in ["pmid", "reactions", "reaction_labels", "ranked_reaction_labels"] } record_jsons.append(record_dict) with open(records_path, "w") as f: json.dump(record_jsons, f) # save statistics execution_stats_dict = data_record_collection.execution_stats.to_json() with open(stats_path, "w") as f: json.dump(execution_stats_dict, f) # score output test_dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split="test").to_pandas() test_dataset = test_dataset.sample(n=250, random_state=seed).to_dict(orient="records") # construct mapping from pmid --> label (field, value) pairs def compute_target_record(entry): reactions_lst = [ reaction.strip().lower().replace("'", "").replace("^", "") for reaction in entry["reactions"].split(",") ] label_dict = {"ranked_reaction_labels": reactions_lst} return label_dict label_fields_to_values = { entry["pmid"]: compute_target_record(entry) for entry in test_dataset } def rank_precision_at_k(preds: list, targets: list, k: int): if preds is None: return 0.0 # lower-case each list preds = [pred.lower().replace("'", "").replace("^", "") for pred in preds] targets = set([target.lower().replace("'", "").replace("^", "") for target in targets]) # compute rank-precision at k rn = len(targets) denom = min(k, rn) total = 0.0 for i in range(k): total += preds[i] in targets if i < len(preds) else 0.0 return total / denom def compute_avg_rp_at_k(records, k=5): total_rp_at_k = 0 bad = 0 for record in records: pmid = record['pmid'] preds = record['ranked_reaction_labels'] targets = label_fields_to_values[pmid]['ranked_reaction_labels'] try: total_rp_at_k += rank_precision_at_k(preds, targets, k) except Exception: bad += 1 return total_rp_at_k / len(records), bad rp_at_k, bad = compute_avg_rp_at_k(record_jsons, k=5) final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0] final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id] stats_dict = { "rp@5": rp_at_k, "optimization_time": data_record_collection.execution_stats.optimization_time, "optimization_cost": data_record_collection.execution_stats.optimization_cost, "plan_execution_time": data_record_collection.execution_stats.plan_execution_time, "plan_execution_cost": data_record_collection.execution_stats.plan_execution_cost, "total_execution_time": data_record_collection.execution_stats.total_execution_time, "total_execution_cost": data_record_collection.execution_stats.total_execution_cost, "plan_str": final_plan_str, } with open(f"ablation-data/{exp_name}-metrics.json", "w") as f: json.dump(stats_dict, f) print(f"bad: {bad}") print("-------") print(f"rp@k: {rp_at_k:.5f}") print(f"Optimization time: {data_record_collection.execution_stats.optimization_time}") print(f"Optimization cost: {data_record_collection.execution_stats.optimization_cost}") print(f"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}") print(f"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}") print(f"Total Execution time: {data_record_collection.execution_stats.total_execution_time}") print(f"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}") ================================================ FILE: abacus-research/biodex-demo.py ================================================ import argparse import json import os import time import chromadb import datasets from chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction import palimpzest as pz from palimpzest.constants import Model biodex_entry_cols = [ {"name": "pmid", "type": str, "desc": "The PubMed ID of the medical paper"}, {"name": "title", "type": str, "desc": "The title of the medical paper"}, {"name": "abstract", "type": str, "desc": "The abstract of the medical paper"}, {"name": "fulltext", "type": str, "desc": "The full text of the medical paper, which contains information relevant for creating a drug safety report."}, ] biodex_reactions_cols = [ {"name": "reactions", "type": list[str], "desc": "The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible."}, ] biodex_reaction_labels_cols = [ {"name": "reaction_labels", "type": list[str], "desc": "Official terms for medical conditions listed in `reactions`"}, ] biodex_ranked_reactions_labels_cols = [ {"name": "ranked_reaction_labels", "type": list[str], "desc": "The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs."}, ] class BiodexValidator(pz.Validator): def __init__( self, rp_at_k: int = 5, num_samples: int = 5, shuffle: bool = False, seed: int = 42, ): super().__init__() # read dataset and prepare entries dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split="train").to_pandas() if shuffle: dataset = dataset.sample(n=num_samples, random_state=seed).to_dict(orient="records") else: dataset = dataset.to_dict(orient="records")[:num_samples] # compute mapping from pmid --> label (i.e. reactions list) self.pmid_to_label = self._compute_pmid_to_label(dataset) # store rp_at_k for computing rank-precision at k metric self.k = rp_at_k def _compute_pmid_to_label(self, dataset: list[dict]) -> dict: """Compute the label for a BioDEX report given its entry in the dataset.""" pmid_to_label = {} for entry in dataset: pmid = str(entry["pmid"]) reactions_lst = [ reaction.strip().lower().replace("'", "").replace("^", "") for reaction in entry["reactions"].split(",") ] pmid_to_label[pmid] = reactions_lst return pmid_to_label def rank_precision_at_k(self, preds: list | None, targets: list): if preds is None: return 0.0 try: # lower-case each list preds = [pred.strip().lower().replace("'", "").replace("^", "") for pred in preds] targets = set([target.strip().lower().replace("'", "").replace("^", "") for target in targets]) # compute rank-precision at k rn = len(targets) denom = min(self.k, rn) total = 0.0 for i in range(self.k): total += preds[i] in targets if i < len(preds) else 0.0 return total / denom except Exception: os.makedirs("rp@k-errors", exist_ok=True) ts = time.time() with open(f"rp@k-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def term_recall(self, preds: list | None, targets: list): if preds is None: return 0.0 try: # normalize terms in each list pred_terms = set([ term.strip() for pred in preds for term in pred.lower().replace("'", "").replace("^", "").split(" ") ]) target_terms = ([ term.strip() for target in targets for term in target.lower().replace("'", "").replace("^", "").split(" ") ]) # compute term recall and return intersect = pred_terms.intersection(target_terms) term_recall = len(intersect) / len(target_terms) return term_recall except Exception: os.makedirs("term-recall-eval-errors", exist_ok=True) ts = time.time() with open(f"term-recall-eval-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: field_name = fields[0] if field_name == "reactions": preds = output.get(field_name) targets = self.pmid_to_label[str(input_record["pmid"])] return self.term_recall(preds, targets) elif field_name == "ranked_reaction_labels": preds = output.get(field_name) targets = self.pmid_to_label[str(input_record["pmid"])] return self.rank_precision_at_k(preds, targets) else: raise NotImplementedError(f"Validator.map_score_fn not implemented for field {field_name}.") def topk_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: field_name = fields[0] if field_name == "reaction_labels": preds = output.get(field_name) targets = self.pmid_to_label[input_record["pmid"]] return self.term_recall(preds, targets) else: raise NotImplementedError(f"Validator.topk_score_fn not implemented for field {field_name}.") class BiodexDataset(pz.IterDataset): def __init__( self, rp_at_k: int = 5, num_samples: int = 5, split: str = "test", shuffle: bool = False, seed: int = 42, ): super().__init__(id="biodex", schema=biodex_entry_cols) self.dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split=split).to_pandas() if shuffle: self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient="records") else: self.dataset = self.dataset.to_dict(orient="records")[:num_samples] self.rp_at_k = rp_at_k self.num_samples = num_samples self.shuffle = shuffle self.seed = seed self.split = split def __len__(self): return len(self.dataset) def __getitem__(self, idx: int): # get entry entry = self.dataset[idx] # get input fields pmid = entry["pmid"] title = entry["title"] abstract = entry["abstract"] fulltext = entry["fulltext"] # create item with fields item = {"pmid": pmid, "title": title, "abstract": abstract, "fulltext": fulltext} return item if __name__ == "__main__": # parse arguments parser = argparse.ArgumentParser(description="Run a simple demo") parser.add_argument("--verbose", default=False, action="store_true", help="Print verbose output") parser.add_argument("--progress", default=False, action="store_true", help="Print progress output") parser.add_argument("--constrained", default=False, action="store_true", help="Use constrained objective") parser.add_argument("--gpt4-mini-only", default=False, action="store_true", help="Use only GPT-4o-mini") parser.add_argument( "--execution-strategy", default="parallel", type=str, help="The plan executor to use. One of sequential, pipelined, parallel", ) parser.add_argument( "--sentinel-execution-strategy", default="mab", type=str, help="The sentinel execution strategy to use. One of mab or random", ) parser.add_argument( "--policy", default="maxquality", type=str, help="One of 'mincost', 'mintime', 'maxquality'", ) parser.add_argument( "--val-examples", default=25, type=int, help="Number of validation examples to sample from", ) parser.add_argument( "--model", default="gpt-4o", type=str, help="One of 'gpt-4o', 'gpt-4o-mini', 'llama'", ) parser.add_argument( "--seed", default=42, type=int, help="Seed used to initialize RNG for MAB sampling algorithm", ) parser.add_argument( "--k", default=10, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--j", default=3, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--sample-budget", default=100, type=int, help="Total sample budget in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--exp-name", default=None, type=str, help="The experiment name.", ) parser.add_argument( "--priors-file", default=None, type=str, help="A file with a dictionary mapping physical operator ids to prior belief on their performance", ) parser.add_argument( "--quality", default=None, type=float, help="Quality threshold", ) args = parser.parse_args() # create directory for profiling data os.makedirs("opt-profiling-data", exist_ok=True) verbose = args.verbose progress = args.progress seed = args.seed val_examples = args.val_examples k = args.k j = args.j sample_budget = args.sample_budget execution_strategy = args.execution_strategy sentinel_execution_strategy = args.sentinel_execution_strategy exp_name = ( f"biodex-final-{sentinel_execution_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}" if args.exp_name is None else args.exp_name ) priors = None if args.priors_file is not None: with open(args.priors_file) as f: priors = json.load(f) # set the optimization policy; constraint set to 25% percentile from unconstrained plans policy = pz.MaxQuality() if not args.constrained else pz.MaxQualityAtFixedCost(max_cost=2.250) if args.policy == "mincost": policy = pz.MinCost() elif args.policy == "minlatency": policy = pz.MinTime() elif args.quality is not None and args.policy == "mincostatfixedquality": policy = pz.MinCostAtFixedQuality(min_quality=args.quality) elif args.quality is not None and args.policy == "minlatencyatfixedquality": policy = pz.MinTimeAtFixedQuality(min_quality=args.quality) print(f"USING POLICY: {policy}") if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") # create validator validator = BiodexValidator( rp_at_k=5, num_samples=val_examples, shuffle=True, seed=seed, ) # create train dataset for validator train_dataset = BiodexDataset( split="train", num_samples=val_examples, shuffle=True, seed=seed, ) train_dataset = {train_dataset.id: train_dataset} # load index [text-embedding-3-small] chroma_client = chromadb.PersistentClient(".chroma-biodex") openai_ef = OpenAIEmbeddingFunction( api_key=os.environ["OPENAI_API_KEY"], model_name="text-embedding-3-small", ) index = chroma_client.get_collection("biodex-reaction-terms", embedding_function=openai_ef) def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]: # execute query with embeddings results = index.query(query, n_results=5) # get list of result terms with their cosine similarity scores final_results = [] for query_docs, query_distances in zip(results["documents"], results["distances"]): for doc, dist in zip(query_docs, query_distances): cosine_similarity = 1 - dist final_results.append({"content": doc, "similarity": cosine_similarity}) # sort the results by similarity score sorted_results = sorted(final_results, key=lambda result: result["similarity"], reverse=True) # remove duplicates sorted_results_set = set() final_sorted_results = [] for result in sorted_results: if result["content"] not in sorted_results_set: sorted_results_set.add(result["content"]) final_sorted_results.append(result["content"]) # return the top-k similar results and generation stats return {"reaction_labels": final_sorted_results[:k]} # construct plan plan = BiodexDataset(split="test", num_samples=250, shuffle=True, seed=seed) plan = plan.sem_map(biodex_reactions_cols) plan = plan.sem_topk( index=index, search_func=search_func, search_attr="reactions", output_attrs=biodex_reaction_labels_cols, ) plan = plan.sem_map(biodex_ranked_reactions_labels_cols, depends_on=["title", "abstract", "fulltext", "reaction_labels"]) # set models models = [Model.GPT_4o_MINI] if args.gpt4_mini_only else [ Model.GPT_4o, Model.GPT_4o_MINI, Model.LLAMA3_1_8B, Model.LLAMA3_3_70B, # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments` Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B, ] # execute pz plan config = pz.QueryProcessorConfig( policy=policy, optimizer_strategy="pareto", sentinel_execution_strategy=sentinel_execution_strategy, execution_strategy=execution_strategy, use_final_op_quality=True, max_workers=64, verbose=verbose, available_models=models, allow_bonded_query=True, allow_critic=True, allow_mixtures=True, allow_rag_reduction=True, progress=progress, k=k, j=j, sample_budget=sample_budget, # sample_cost_budget=0.10, seed=seed, exp_name=exp_name, priors=priors, ) data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator) print(data_record_collection.to_df()) data_record_collection.to_df().to_csv(f"opt-profiling-data/{exp_name}-output.csv", index=False) # create filepaths for records and stats records_path = f"opt-profiling-data/{exp_name}-records.json" stats_path = f"opt-profiling-data/{exp_name}-profiling.json" # save record outputs record_jsons = [] for record in data_record_collection: record_dict = record.to_dict() record_dict = { k: v for k, v in record_dict.items() if k in ["pmid", "reactions", "reaction_labels", "ranked_reaction_labels"] } record_jsons.append(record_dict) with open(records_path, "w") as f: json.dump(record_jsons, f) # save statistics execution_stats_dict = data_record_collection.execution_stats.to_json() with open(stats_path, "w") as f: json.dump(execution_stats_dict, f) # score output test_dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split="test").to_pandas() test_dataset = test_dataset.sample(n=250, random_state=seed).to_dict(orient="records") # construct mapping from pmid --> label (field, value) pairs def compute_target_record(entry): reactions_lst = [ reaction.strip().lower().replace("'", "").replace("^", "") for reaction in entry["reactions"].split(",") ] label_dict = {"ranked_reaction_labels": reactions_lst} return label_dict label_fields_to_values = { entry["pmid"]: compute_target_record(entry) for entry in test_dataset } def rank_precision_at_k(preds: list, targets: list, k: int): if preds is None: return 0.0 # lower-case each list preds = [pred.lower().replace("'", "").replace("^", "") for pred in preds] targets = set([target.lower().replace("'", "").replace("^", "") for target in targets]) # compute rank-precision at k rn = len(targets) denom = min(k, rn) total = 0.0 for i in range(k): total += preds[i] in targets if i < len(preds) else 0.0 return total / denom def compute_avg_rp_at_k(records, k=5): total_rp_at_k = 0 bad = 0 for record in records: pmid = record['pmid'] preds = record['ranked_reaction_labels'] targets = label_fields_to_values[pmid]['ranked_reaction_labels'] try: total_rp_at_k += rank_precision_at_k(preds, targets, k) except Exception: bad += 1 return total_rp_at_k / len(records), bad rp_at_k, bad = compute_avg_rp_at_k(record_jsons, k=5) final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0] final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id] stats_dict = { "rp@5": rp_at_k, "optimization_time": data_record_collection.execution_stats.optimization_time, "optimization_cost": data_record_collection.execution_stats.optimization_cost, "plan_execution_time": data_record_collection.execution_stats.plan_execution_time, "plan_execution_cost": data_record_collection.execution_stats.plan_execution_cost, "total_execution_time": data_record_collection.execution_stats.total_execution_time, "total_execution_cost": data_record_collection.execution_stats.total_execution_cost, "plan_str": final_plan_str, } with open(f"opt-profiling-data/{exp_name}-metrics.json", "w") as f: json.dump(stats_dict, f) print(f"bad: {bad}") print("-------") print(f"rp@k: {rp_at_k:.5f}") print(f"Optimization time: {data_record_collection.execution_stats.optimization_time}") print(f"Optimization cost: {data_record_collection.execution_stats.optimization_cost}") print(f"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}") print(f"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}") print(f"Total Execution time: {data_record_collection.execution_stats.total_execution_time}") print(f"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}") ================================================ FILE: abacus-research/biodex-max-quality-at-cost.py ================================================ import argparse import json import os import time import chromadb import datasets from chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction # from ragatouille import RAGPretrainedModel import palimpzest as pz from palimpzest.constants import Model from palimpzest.policy import MaxQuality, MaxQualityAtFixedCost biodex_entry_cols = [ {"name": "pmid", "type": str, "desc": "The PubMed ID of the medical paper"}, {"name": "title", "type": str, "desc": "The title of the medical paper"}, {"name": "abstract", "type": str, "desc": "The abstract of the medical paper"}, {"name": "fulltext", "type": str, "desc": "The full text of the medical paper, which contains information relevant for creating a drug safety report."}, ] biodex_reactions_cols = [ {"name": "reactions", "type": list[str], "desc": "The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible."}, ] biodex_reaction_labels_cols = [ {"name": "reaction_labels", "type": list[str], "desc": "Official terms for medical conditions listed in `reactions`"}, ] biodex_ranked_reactions_labels_cols = [ {"name": "ranked_reaction_labels", "type": list[str], "desc": "The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs."}, ] class BiodexValidator(pz.Validator): def __init__( self, rp_at_k: int = 5, num_samples: int = 5, shuffle: bool = False, seed: int = 42, ): super().__init__() # read dataset and prepare entries dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split="train").to_pandas() if shuffle: dataset = dataset.sample(n=num_samples, random_state=seed).to_dict(orient="records") else: dataset = dataset.to_dict(orient="records")[:num_samples] # compute mapping from pmid --> label (i.e. reactions list) self.pmid_to_label = self._compute_pmid_to_label(dataset) # store rp_at_k for computing rank-precision at k metric self.k = rp_at_k def _compute_pmid_to_label(self, dataset: list[dict]) -> dict: """Compute the label for a BioDEX report given its entry in the dataset.""" pmid_to_label = {} for entry in dataset: pmid = str(entry["pmid"]) reactions_lst = [ reaction.strip().lower().replace("'", "").replace("^", "") for reaction in entry["reactions"].split(",") ] pmid_to_label[pmid] = reactions_lst return pmid_to_label def rank_precision_at_k(self, preds: list | None, targets: list): if preds is None: return 0.0 try: # lower-case each list preds = [pred.strip().lower().replace("'", "").replace("^", "") for pred in preds] targets = set([target.strip().lower().replace("'", "").replace("^", "") for target in targets]) # compute rank-precision at k rn = len(targets) denom = min(self.k, rn) total = 0.0 for i in range(self.k): total += preds[i] in targets if i < len(preds) else 0.0 return total / denom except Exception: os.makedirs("rp@k-errors", exist_ok=True) ts = time.time() with open(f"rp@k-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def term_recall(self, preds: list | None, targets: list): if preds is None: return 0.0 try: # normalize terms in each list pred_terms = set([ term.strip() for pred in preds for term in pred.lower().replace("'", "").replace("^", "").split(" ") ]) target_terms = ([ term.strip() for target in targets for term in target.lower().replace("'", "").replace("^", "").split(" ") ]) # compute term recall and return intersect = pred_terms.intersection(target_terms) term_recall = len(intersect) / len(target_terms) return term_recall except Exception: os.makedirs("term-recall-eval-errors", exist_ok=True) ts = time.time() with open(f"term-recall-eval-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: field_name = fields[0] if field_name == "reactions": preds = output.get(field_name) targets = self.pmid_to_label[str(input_record["pmid"])] return self.term_recall(preds, targets) elif field_name == "ranked_reaction_labels": preds = output.get(field_name) targets = self.pmid_to_label[str(input_record["pmid"])] return self.rank_precision_at_k(preds, targets) else: raise NotImplementedError(f"Validator.map_score_fn not implemented for field {field_name}.") def topk_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: field_name = fields[0] if field_name == "reaction_labels": preds = output.get(field_name) targets = self.pmid_to_label[input_record["pmid"]] return self.term_recall(preds, targets) else: raise NotImplementedError(f"Validator.topk_score_fn not implemented for field {field_name}.") class BiodexDataset(pz.IterDataset): def __init__( self, rp_at_k: int = 5, num_samples: int = 5, split: str = "test", shuffle: bool = False, seed: int = 42, ): super().__init__(id="biodex", schema=biodex_entry_cols) self.dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split=split).to_pandas() if shuffle: self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient="records") else: self.dataset = self.dataset.to_dict(orient="records")[:num_samples] self.rp_at_k = rp_at_k self.num_samples = num_samples self.shuffle = shuffle self.seed = seed self.split = split def __len__(self): return len(self.dataset) def __getitem__(self, idx: int): # get entry entry = self.dataset[idx] # get input fields pmid = entry["pmid"] title = entry["title"] abstract = entry["abstract"] fulltext = entry["fulltext"] # create item with fields item = {"fields": {}, "labels": {}, "score_fn": {}} item["fields"]["pmid"] = pmid item["fields"]["title"] = title item["fields"]["abstract"] = abstract item["fields"]["fulltext"] = fulltext return item if __name__ == "__main__": # parse arguments parser = argparse.ArgumentParser(description="Run a simple demo") parser.add_argument("--verbose", default=False, action="store_true", help="Print verbose output") parser.add_argument("--progress", default=False, action="store_true", help="Print progress output") parser.add_argument( "--execution-strategy", default="parallel", type=str, help="The plan executor to use. One of sequential, pipelined, parallel", ) parser.add_argument( "--sentinel-execution-strategy", default="mab", type=str, help="The sentinel execution strategy to use. One of mab or random", ) parser.add_argument( "--optimizer-strategy", default="pareto", type=str, help="The optimizer to use. One of pareto or greedy", ) parser.add_argument( "--val-examples", default=30, type=int, help="Number of validation examples to sample from", ) parser.add_argument( "--seed", default=42, type=int, help="Seed used to initialize RNG for MAB sampling algorithm", ) parser.add_argument( "--k", default=10, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--j", default=3, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--sample-budget", default=100, type=int, help="Total sample budget in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--cost", default=1.0, type=float, help="The cost budget for the optimization", ) parser.add_argument( "--exp-name", default=None, type=str, help="The experiment name.", ) parser.add_argument( "--priors-file", default=None, type=str, help="A file with a dictionary mapping physical operator ids to prior belief on their performance", ) args = parser.parse_args() # create directory for profiling data os.makedirs("max-quality-at-cost-data", exist_ok=True) verbose = args.verbose progress = args.progress seed = args.seed val_examples = args.val_examples k = args.k j = args.j sample_budget = args.sample_budget execution_strategy = args.execution_strategy sentinel_execution_strategy = args.sentinel_execution_strategy optimizer_strategy = args.optimizer_strategy cost = args.cost exp_name = ( f"biodex-strategy-{optimizer_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}" if args.exp_name is None else args.exp_name ) priors = None if args.priors_file is not None: with open(args.priors_file) as f: priors = json.load(f) print(f"EXPERIMENT NAME: {exp_name}") if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") # create validator validator = BiodexValidator( rp_at_k=5, num_samples=val_examples, shuffle=True, seed=seed, ) # create validation data source train_dataset = BiodexDataset( split="train", num_samples=val_examples, shuffle=True, seed=seed, ) train_dataset = {train_dataset.id: train_dataset} # load index [text-embedding-3-small] chroma_client = chromadb.PersistentClient(".chroma-biodex") openai_ef = OpenAIEmbeddingFunction( api_key=os.environ["OPENAI_API_KEY"], model_name="text-embedding-3-small", ) index = chroma_client.get_collection("biodex-reaction-terms", embedding_function=openai_ef) def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]: # execute query with embeddings results = index.query(query, n_results=5) # get list of result terms with their cosine similarity scores final_results = [] for query_docs, query_distances in zip(results["documents"], results["distances"]): for doc, dist in zip(query_docs, query_distances): cosine_similarity = 1 - dist final_results.append({"content": doc, "similarity": cosine_similarity}) # sort the results by similarity score sorted_results = sorted(final_results, key=lambda result: result["similarity"], reverse=True) # remove duplicates sorted_results_set = set() final_sorted_results = [] for result in sorted_results: if result["content"] not in sorted_results_set: sorted_results_set.add(result["content"]) final_sorted_results.append(result["content"]) # return the top-k similar results and generation stats return {"reaction_labels": final_sorted_results[:k]} # construct plan plan = BiodexDataset(split="test", num_samples=250, shuffle=True, seed=seed) plan = plan.sem_map(biodex_reactions_cols) plan = plan.sem_topk( index=index, search_func=search_func, search_attr="reactions", output_attrs=biodex_reaction_labels_cols, ) plan = plan.sem_map(biodex_ranked_reactions_labels_cols, depends_on=["title", "abstract", "fulltext", "reaction_labels"]) # set policy policy = MaxQualityAtFixedCost(max_cost=cost) if cost < 999 else MaxQuality() # execute pz plan config = pz.QueryProcessorConfig( policy=policy, optimizer_strategy=optimizer_strategy, sentinel_execution_strategy=sentinel_execution_strategy, execution_strategy=execution_strategy, use_final_op_quality=True, max_workers=64, verbose=verbose, available_models=[ Model.GPT_4o, Model.GPT_4o_MINI, Model.LLAMA3_1_8B, Model.LLAMA3_3_70B, # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments` Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B, ], allow_bonded_query=True, allow_critic=True, allow_mixtures=True, allow_rag_reduction=True, progress=progress, k=k, j=j, sample_budget=sample_budget, seed=seed, exp_name=exp_name, priors=priors, ) data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator) print(data_record_collection.to_df()) data_record_collection.to_df().to_csv(f"max-quality-at-cost-data/{exp_name}-output.csv", index=False) # create filepaths for records and stats records_path = f"max-quality-at-cost-data/{exp_name}-records.json" stats_path = f"max-quality-at-cost-data/{exp_name}-profiling.json" # save record outputs record_jsons = [] for record in data_record_collection: record_dict = record.to_dict() record_dict = { k: v for k, v in record_dict.items() if k in ["pmid", "reactions", "reaction_labels", "ranked_reaction_labels"] } record_jsons.append(record_dict) with open(records_path, "w") as f: json.dump(record_jsons, f) # save statistics execution_stats_dict = data_record_collection.execution_stats.to_json() with open(stats_path, "w") as f: json.dump(execution_stats_dict, f) # score output test_dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split="test").to_pandas() test_dataset = test_dataset.sample(n=250, random_state=seed).to_dict(orient="records") # construct mapping from pmid --> label (field, value) pairs def compute_target_record(entry): reactions_lst = [ reaction.strip().lower().replace("'", "").replace("^", "") for reaction in entry["reactions"].split(",") ] label_dict = {"ranked_reaction_labels": reactions_lst} return label_dict label_fields_to_values = { entry["pmid"]: compute_target_record(entry) for entry in test_dataset } def rank_precision_at_k(preds: list, targets: list, k: int): if preds is None: return 0.0 # lower-case each list preds = [pred.lower().replace("'", "").replace("^", "") for pred in preds] targets = set([target.lower().replace("'", "").replace("^", "") for target in targets]) # compute rank-precision at k rn = len(targets) denom = min(k, rn) total = 0.0 for i in range(k): total += preds[i] in targets if i < len(preds) else 0.0 return total / denom def compute_avg_rp_at_k(records, k=5): total_rp_at_k, bad = 0, 0 for record in records: pmid = record['pmid'] preds = record['ranked_reaction_labels'] targets = label_fields_to_values[pmid]['ranked_reaction_labels'] try: total_rp_at_k += rank_precision_at_k(preds, targets, k) except Exception: print(f"Error computing rank precision at k for record with pmid {pmid}") bad += 1 return total_rp_at_k / len(records), bad rp_at_k, failed = compute_avg_rp_at_k(record_jsons, k=5) final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0] final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id] stats_dict = { "rp@5": rp_at_k, "failed": failed, "optimization_time": data_record_collection.execution_stats.optimization_time, "optimization_cost": data_record_collection.execution_stats.optimization_cost, "plan_execution_time": data_record_collection.execution_stats.plan_execution_time, "plan_execution_cost": data_record_collection.execution_stats.plan_execution_cost, "total_execution_time": data_record_collection.execution_stats.total_execution_time, "total_execution_cost": data_record_collection.execution_stats.total_execution_cost, "plan_str": final_plan_str, } with open(f"max-quality-at-cost-data/{exp_name}-metrics.json", "w") as f: json.dump(stats_dict, f) print(f"rp@k: {rp_at_k:.5f}") print(f"failed: {failed}") print(f"Optimization time: {data_record_collection.execution_stats.optimization_time}") print(f"Optimization cost: {data_record_collection.execution_stats.optimization_cost}") print(f"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}") print(f"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}") print(f"Total Execution time: {data_record_collection.execution_stats.total_execution_time}") print(f"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}") ================================================ FILE: abacus-research/biodex-min-at-fixed-quality.py ================================================ import argparse import json import os import time import chromadb import datasets from chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction # from ragatouille import RAGPretrainedModel import palimpzest as pz from palimpzest.constants import Model biodex_entry_cols = [ {"name": "pmid", "type": str, "desc": "The PubMed ID of the medical paper"}, {"name": "title", "type": str, "desc": "The title of the medical paper"}, {"name": "abstract", "type": str, "desc": "The abstract of the medical paper"}, {"name": "fulltext", "type": str, "desc": "The full text of the medical paper, which contains information relevant for creating a drug safety report."}, ] biodex_reactions_cols = [ {"name": "reactions", "type": list[str], "desc": "The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible."}, ] biodex_reaction_labels_cols = [ {"name": "reaction_labels", "type": list[str], "desc": "Official terms for medical conditions listed in `reactions`"}, ] biodex_ranked_reactions_labels_cols = [ {"name": "ranked_reaction_labels", "type": list[str], "desc": "The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs."}, ] class BiodexValidator(pz.Validator): def __init__( self, rp_at_k: int = 5, num_samples: int = 5, shuffle: bool = False, seed: int = 42, ): super().__init__() # read dataset and prepare entries dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split="train").to_pandas() if shuffle: dataset = dataset.sample(n=num_samples, random_state=seed).to_dict(orient="records") else: dataset = dataset.to_dict(orient="records")[:num_samples] # compute mapping from pmid --> label (i.e. reactions list) self.pmid_to_label = self._compute_pmid_to_label(dataset) # store rp_at_k for computing rank-precision at k metric self.k = rp_at_k def _compute_pmid_to_label(self, dataset: list[dict]) -> dict: """Compute the label for a BioDEX report given its entry in the dataset.""" pmid_to_label = {} for entry in dataset: pmid = str(entry["pmid"]) reactions_lst = [ reaction.strip().lower().replace("'", "").replace("^", "") for reaction in entry["reactions"].split(",") ] pmid_to_label[pmid] = reactions_lst return pmid_to_label def rank_precision_at_k(self, preds: list | None, targets: list): if preds is None: return 0.0 try: # lower-case each list preds = [pred.strip().lower().replace("'", "").replace("^", "") for pred in preds] targets = set([target.strip().lower().replace("'", "").replace("^", "") for target in targets]) # compute rank-precision at k rn = len(targets) denom = min(self.k, rn) total = 0.0 for i in range(self.k): total += preds[i] in targets if i < len(preds) else 0.0 return total / denom except Exception: os.makedirs("rp@k-errors", exist_ok=True) ts = time.time() with open(f"rp@k-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def term_recall(self, preds: list | None, targets: list): if preds is None: return 0.0 try: # normalize terms in each list pred_terms = set([ term.strip() for pred in preds for term in pred.lower().replace("'", "").replace("^", "").split(" ") ]) target_terms = ([ term.strip() for target in targets for term in target.lower().replace("'", "").replace("^", "").split(" ") ]) # compute term recall and return intersect = pred_terms.intersection(target_terms) term_recall = len(intersect) / len(target_terms) return term_recall except Exception: os.makedirs("term-recall-eval-errors", exist_ok=True) ts = time.time() with open(f"term-recall-eval-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: field_name = fields[0] if field_name == "reactions": preds = output.get(field_name) targets = self.pmid_to_label[str(input_record["pmid"])] return self.term_recall(preds, targets) elif field_name == "ranked_reaction_labels": preds = output.get(field_name) targets = self.pmid_to_label[str(input_record["pmid"])] return self.rank_precision_at_k(preds, targets) else: raise NotImplementedError(f"Validator.map_score_fn not implemented for field {field_name}.") def topk_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: field_name = fields[0] if field_name == "reaction_labels": preds = output.get(field_name) targets = self.pmid_to_label[input_record["pmid"]] return self.term_recall(preds, targets) else: raise NotImplementedError(f"Validator.topk_score_fn not implemented for field {field_name}.") class BiodexDataset(pz.IterDataset): def __init__( self, rp_at_k: int = 5, num_samples: int = 5, split: str = "test", shuffle: bool = False, seed: int = 42, ): super().__init__(id="biodex", schema=biodex_entry_cols) self.dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split=split).to_pandas() if shuffle: self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient="records") else: self.dataset = self.dataset.to_dict(orient="records")[:num_samples] self.rp_at_k = rp_at_k self.num_samples = num_samples self.shuffle = shuffle self.seed = seed self.split = split def __len__(self): return len(self.dataset) def __getitem__(self, idx: int): # get entry entry = self.dataset[idx] # get input fields pmid = entry["pmid"] title = entry["title"] abstract = entry["abstract"] fulltext = entry["fulltext"] # create item with fields item = {"pmid": pmid, "title": title, "abstract": abstract, "fulltext": fulltext} return item if __name__ == "__main__": # parse arguments parser = argparse.ArgumentParser(description="Run a simple demo") parser.add_argument("--verbose", default=False, action="store_true", help="Print verbose output") parser.add_argument("--progress", default=False, action="store_true", help="Print progress output") parser.add_argument( "--execution-strategy", default="parallel", type=str, help="The plan executor to use. One of sequential, pipelined, parallel", ) parser.add_argument( "--sentinel-execution-strategy", default="mab", type=str, help="The sentinel execution strategy to use. One of mab or random", ) parser.add_argument( "--optimizer-strategy", default="pareto", type=str, help="The optimizer to use. One of pareto or greedy", ) parser.add_argument( "--val-examples", default=30, type=int, help="Number of validation examples to sample from", ) parser.add_argument( "--seed", default=42, type=int, help="Seed used to initialize RNG for MAB sampling algorithm", ) parser.add_argument( "--k", default=10, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--j", default=3, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--sample-budget", default=100, type=int, help="Total sample budget in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--metric", default=None, type=str, help="whether to minimize latency or cost", ) parser.add_argument( "--exp-name", default=None, type=str, help="The experiment name.", ) parser.add_argument( "--priors-file", default=None, type=str, help="A file with a dictionary mapping physical operator ids to prior belief on their performance", ) args = parser.parse_args() assert args.metric in ["cost", "latency"], "metric must be one of cost or latency" metric = args.metric # create directory for profiling data os.makedirs(f"min-{metric}-at-quality-data", exist_ok=True) verbose = args.verbose progress = args.progress seed = args.seed val_examples = args.val_examples k = args.k j = args.j sample_budget = args.sample_budget execution_strategy = args.execution_strategy sentinel_execution_strategy = args.sentinel_execution_strategy optimizer_strategy = args.optimizer_strategy exp_name = ( f"biodex-min-{metric}-strategy-{optimizer_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}" if args.exp_name is None else args.exp_name ) priors = None if args.priors_file is not None: with open(args.priors_file) as f: priors = json.load(f) print(f"EXPERIMENT NAME: {exp_name}") if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") # create validator validator = BiodexValidator( rp_at_k=5, num_samples=val_examples, shuffle=True, seed=seed, ) # create validation data source train_dataset = BiodexDataset( split="train", num_samples=val_examples, shuffle=True, seed=seed, ) train_dataset = {train_dataset.id: train_dataset} # load index [text-embedding-3-small] chroma_client = chromadb.PersistentClient(".chroma-biodex") openai_ef = OpenAIEmbeddingFunction( api_key=os.environ["OPENAI_API_KEY"], model_name="text-embedding-3-small", ) index = chroma_client.get_collection("biodex-reaction-terms", embedding_function=openai_ef) def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]: # execute query with embeddings results = index.query(query, n_results=5) # get list of result terms with their cosine similarity scores final_results = [] for query_docs, query_distances in zip(results["documents"], results["distances"]): for doc, dist in zip(query_docs, query_distances): cosine_similarity = 1 - dist final_results.append({"content": doc, "similarity": cosine_similarity}) # sort the results by similarity score sorted_results = sorted(final_results, key=lambda result: result["similarity"], reverse=True) # remove duplicates sorted_results_set = set() final_sorted_results = [] for result in sorted_results: if result["content"] not in sorted_results_set: sorted_results_set.add(result["content"]) final_sorted_results.append(result["content"]) # return the top-k similar results and generation stats return {"reaction_labels": final_sorted_results[:k]} # construct plan plan = BiodexDataset(split="test", num_samples=250, shuffle=True, seed=seed) plan = plan.sem_map(biodex_reactions_cols) plan = plan.sem_topk( index=index, search_func=search_func, search_attr="reactions", output_attrs=biodex_reaction_labels_cols, ) plan = plan.sem_map(biodex_ranked_reactions_labels_cols, depends_on=["title", "abstract", "fulltext", "reaction_labels"]) # set policy policy = pz.MinCostAtFixedQuality(min_quality=0.216) if metric == "cost" else pz.MinTimeAtFixedQuality(min_quality=0.216) # execute pz plan config = pz.QueryProcessorConfig( policy=policy, optimizer_strategy=optimizer_strategy, sentinel_execution_strategy=sentinel_execution_strategy, execution_strategy=execution_strategy, use_final_op_quality=True, max_workers=64, verbose=verbose, available_models=[ Model.GPT_4o_MINI, ], allow_bonded_query=True, allow_critic=True, allow_mixtures=True, allow_rag_reduction=True, progress=progress, k=k, j=j, sample_budget=sample_budget, seed=seed, exp_name=exp_name, priors=priors, ) data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator) print(data_record_collection.to_df()) data_record_collection.to_df().to_csv(f"min-{metric}-at-quality-data/{exp_name}-output.csv", index=False) # create filepaths for records and stats records_path = f"min-{metric}-at-quality-data/{exp_name}-records.json" stats_path = f"min-{metric}-at-quality-data/{exp_name}-profiling.json" # save record outputs record_jsons = [] for record in data_record_collection: record_dict = record.to_dict() record_dict = { k: v for k, v in record_dict.items() if k in ["pmid", "reactions", "reaction_labels", "ranked_reaction_labels"] } record_jsons.append(record_dict) with open(records_path, "w") as f: json.dump(record_jsons, f) # save statistics execution_stats_dict = data_record_collection.execution_stats.to_json() with open(stats_path, "w") as f: json.dump(execution_stats_dict, f) # score output test_dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split="test").to_pandas() test_dataset = test_dataset.sample(n=250, random_state=seed).to_dict(orient="records") # construct mapping from pmid --> label (field, value) pairs def compute_target_record(entry): reactions_lst = [ reaction.strip().lower().replace("'", "").replace("^", "") for reaction in entry["reactions"].split(",") ] label_dict = {"ranked_reaction_labels": reactions_lst} return label_dict label_fields_to_values = { entry["pmid"]: compute_target_record(entry) for entry in test_dataset } def rank_precision_at_k(preds: list, targets: list, k: int): if preds is None: return 0.0 # lower-case each list preds = [pred.lower().replace("'", "").replace("^", "") for pred in preds] targets = set([target.lower().replace("'", "").replace("^", "") for target in targets]) # compute rank-precision at k rn = len(targets) denom = min(k, rn) total = 0.0 for i in range(k): total += preds[i] in targets if i < len(preds) else 0.0 return total / denom def compute_avg_rp_at_k(records, k=5): total_rp_at_k, bad = 0, 0 for record in records: pmid = record['pmid'] preds = record['ranked_reaction_labels'] targets = label_fields_to_values[pmid]['ranked_reaction_labels'] try: total_rp_at_k += rank_precision_at_k(preds, targets, k) except Exception: print(f"Error computing rank precision at k for record with pmid {pmid}") bad += 1 return total_rp_at_k / len(records), bad rp_at_k, failed = compute_avg_rp_at_k(record_jsons, k=5) final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0] final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id] stats_dict = { "rp@5": rp_at_k, "failed": failed, "optimization_time": data_record_collection.execution_stats.optimization_time, "optimization_cost": data_record_collection.execution_stats.optimization_cost, "plan_execution_time": data_record_collection.execution_stats.plan_execution_time, "plan_execution_cost": data_record_collection.execution_stats.plan_execution_cost, "total_execution_time": data_record_collection.execution_stats.total_execution_time, "total_execution_cost": data_record_collection.execution_stats.total_execution_cost, "plan_str": final_plan_str, } with open(f"min-{metric}-at-quality-data/{exp_name}-metrics.json", "w") as f: json.dump(stats_dict, f) print(f"rp@k: {rp_at_k:.5f}") print(f"failed: {failed}") print(f"Optimization time: {data_record_collection.execution_stats.optimization_time}") print(f"Optimization cost: {data_record_collection.execution_stats.optimization_cost}") print(f"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}") print(f"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}") print(f"Total Execution time: {data_record_collection.execution_stats.total_execution_time}") print(f"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}") ================================================ FILE: abacus-research/biodex-pareto-cascades.py ================================================ import argparse import json import os import time import chromadb import datasets from chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction import palimpzest as pz from palimpzest.constants import Model from palimpzest.policy import MaxQualityAtFixedCost biodex_entry_cols = [ {"name": "pmid", "type": str, "desc": "The PubMed ID of the medical paper"}, {"name": "title", "type": str, "desc": "The title of the medical paper"}, {"name": "abstract", "type": str, "desc": "The abstract of the medical paper"}, {"name": "fulltext", "type": str, "desc": "The full text of the medical paper, which contains information relevant for creating a drug safety report."}, ] biodex_reactions_cols = [ {"name": "reactions", "type": list[str], "desc": "The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible."}, ] biodex_reaction_labels_cols = [ {"name": "reaction_labels", "type": list[str], "desc": "Official terms for medical conditions listed in `reactions`"}, ] biodex_ranked_reactions_labels_cols = [ {"name": "ranked_reaction_labels", "type": list[str], "desc": "The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs."}, ] class BiodexValidator(pz.Validator): def __init__( self, rp_at_k: int = 5, num_samples: int = 5, shuffle: bool = False, seed: int = 42, ): super().__init__() # read dataset and prepare entries dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split="train").to_pandas() if shuffle: dataset = dataset.sample(n=num_samples, random_state=seed).to_dict(orient="records") else: dataset = dataset.to_dict(orient="records")[:num_samples] # compute mapping from pmid --> label (i.e. reactions list) self.pmid_to_label = self._compute_pmid_to_label(dataset) # store rp_at_k for computing rank-precision at k metric self.k = rp_at_k def _compute_pmid_to_label(self, dataset: list[dict]) -> dict: """Compute the label for a BioDEX report given its entry in the dataset.""" pmid_to_label = {} for entry in dataset: pmid = str(entry["pmid"]) reactions_lst = [ reaction.strip().lower().replace("'", "").replace("^", "") for reaction in entry["reactions"].split(",") ] pmid_to_label[pmid] = reactions_lst return pmid_to_label def rank_precision_at_k(self, preds: list | None, targets: list): if preds is None: return 0.0 try: # lower-case each list preds = [pred.strip().lower().replace("'", "").replace("^", "") for pred in preds] targets = set([target.strip().lower().replace("'", "").replace("^", "") for target in targets]) # compute rank-precision at k rn = len(targets) denom = min(self.k, rn) total = 0.0 for i in range(self.k): total += preds[i] in targets if i < len(preds) else 0.0 return total / denom except Exception: os.makedirs("rp@k-errors", exist_ok=True) ts = time.time() with open(f"rp@k-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def term_recall(self, preds: list | None, targets: list): if preds is None: return 0.0 try: # normalize terms in each list pred_terms = set([ term.strip() for pred in preds for term in pred.lower().replace("'", "").replace("^", "").split(" ") ]) target_terms = ([ term.strip() for target in targets for term in target.lower().replace("'", "").replace("^", "").split(" ") ]) # compute term recall and return intersect = pred_terms.intersection(target_terms) term_recall = len(intersect) / len(target_terms) return term_recall except Exception: os.makedirs("term-recall-eval-errors", exist_ok=True) ts = time.time() with open(f"term-recall-eval-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: field_name = fields[0] if field_name == "reactions": preds = output.get(field_name) targets = self.pmid_to_label[str(input_record["pmid"])] return self.term_recall(preds, targets) elif field_name == "ranked_reaction_labels": preds = output.get(field_name) targets = self.pmid_to_label[str(input_record["pmid"])] return self.rank_precision_at_k(preds, targets) else: raise NotImplementedError(f"Validator.map_score_fn not implemented for field {field_name}.") def topk_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: field_name = fields[0] if field_name == "reaction_labels": preds = output.get(field_name) targets = self.pmid_to_label[input_record["pmid"]] return self.term_recall(preds, targets) else: raise NotImplementedError(f"Validator.topk_score_fn not implemented for field {field_name}.") class BiodexDataset(pz.IterDataset): def __init__( self, rp_at_k: int = 5, num_samples: int = 5, split: str = "test", shuffle: bool = False, seed: int = 42, ): super().__init__(id="biodex", schema=biodex_entry_cols) self.dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split=split).to_pandas() if shuffle: self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient="records") else: self.dataset = self.dataset.to_dict(orient="records")[:num_samples] self.rp_at_k = rp_at_k self.num_samples = num_samples self.shuffle = shuffle self.seed = seed self.split = split def __len__(self): return len(self.dataset) def __getitem__(self, idx: int): # get entry entry = self.dataset[idx] # get input fields pmid = entry["pmid"] title = entry["title"] abstract = entry["abstract"] fulltext = entry["fulltext"] # create item with fields item = {"fields": {}, "labels": {}, "score_fn": {}} item["fields"]["pmid"] = pmid item["fields"]["title"] = title item["fields"]["abstract"] = abstract item["fields"]["fulltext"] = fulltext return item if __name__ == "__main__": # parse arguments parser = argparse.ArgumentParser(description="Run a simple demo") parser.add_argument("--verbose", default=False, action="store_true", help="Print verbose output") parser.add_argument("--progress", default=False, action="store_true", help="Print progress output") parser.add_argument("--constrained", default=False, action="store_true", help="Use constrained objective") parser.add_argument( "--execution-strategy", default="parallel", type=str, help="The plan executor to use. One of sequential, pipelined, parallel", ) parser.add_argument( "--sentinel-execution-strategy", default="mab", type=str, help="The sentinel execution strategy to use. One of mab or random", ) parser.add_argument( "--optimizer-strategy", default="pareto", type=str, help="The optimizer to use. One of pareto or greedy", ) parser.add_argument( "--val-examples", default=30, type=int, help="Number of validation examples to sample from", ) parser.add_argument( "--seed", default=42, type=int, help="Seed used to initialize RNG for MAB sampling algorithm", ) parser.add_argument( "--k", default=10, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--j", default=3, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--sample-budget", default=100, type=int, help="Total sample budget in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--cost", default=1.0, type=float, help="The cost budget for the optimization", ) parser.add_argument( "--exp-name", default=None, type=str, help="The experiment name.", ) parser.add_argument( "--priors-file", default=None, type=str, help="A file with a dictionary mapping physical operator ids to prior belief on their performance", ) args = parser.parse_args() # create directory for profiling data os.makedirs("pareto-cascades-data", exist_ok=True) verbose = args.verbose progress = args.progress seed = args.seed val_examples = args.val_examples k = args.k j = args.j sample_budget = args.sample_budget execution_strategy = args.execution_strategy sentinel_execution_strategy = args.sentinel_execution_strategy optimizer_strategy = args.optimizer_strategy cost = args.cost exp_name = ( f"biodex-strategy-{optimizer_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}" if args.exp_name is None else args.exp_name ) priors = None if args.priors_file is not None: with open(args.priors_file) as f: priors = json.load(f) print(f"EXPERIMENT NAME: {exp_name}") if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") # create validator validator = BiodexValidator( rp_at_k=5, num_samples=val_examples, shuffle=True, seed=seed, ) # create validation data source train_dataset = BiodexDataset( split="train", num_samples=val_examples, shuffle=True, seed=seed, ) train_dataset = {train_dataset.id: train_dataset} # load index [text-embedding-3-small] chroma_client = chromadb.PersistentClient(".chroma-biodex") openai_ef = OpenAIEmbeddingFunction( api_key=os.environ["OPENAI_API_KEY"], model_name="text-embedding-3-small", ) index = chroma_client.get_collection("biodex-reaction-terms", embedding_function=openai_ef) def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]: # execute query with embeddings results = index.query(query, n_results=5) # get list of result terms with their cosine similarity scores final_results = [] for query_docs, query_distances in zip(results["documents"], results["distances"]): for doc, dist in zip(query_docs, query_distances): cosine_similarity = 1 - dist final_results.append({"content": doc, "similarity": cosine_similarity}) # sort the results by similarity score sorted_results = sorted(final_results, key=lambda result: result["similarity"], reverse=True) # remove duplicates sorted_results_set = set() final_sorted_results = [] for result in sorted_results: if result["content"] not in sorted_results_set: sorted_results_set.add(result["content"]) final_sorted_results.append(result["content"]) # return the top-k similar results and generation stats return {"reaction_labels": final_sorted_results[:k]} # construct plan plan = BiodexDataset(split="test", num_samples=250, shuffle=True, seed=seed) plan = plan.sem_map(biodex_reactions_cols) plan = plan.sem_topk( index=index, search_func=search_func, search_attr="reactions", output_attrs=biodex_reaction_labels_cols, ) plan = plan.sem_map(biodex_ranked_reactions_labels_cols, depends_on=["title", "abstract", "fulltext", "reaction_labels"]) # execute pz plan config = pz.QueryProcessorConfig( policy=MaxQualityAtFixedCost(max_cost=cost), optimizer_strategy=optimizer_strategy, sentinel_execution_strategy=sentinel_execution_strategy, execution_strategy=execution_strategy, use_final_op_quality=True, max_workers=64, verbose=verbose, available_models=[ Model.GPT_4o_MINI, Model.LLAMA3_2_3B, Model.LLAMA3_1_8B, Model.LLAMA3_3_70B, # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments` Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B, ], allow_bonded_query=True, allow_critic=True, allow_mixtures=True, allow_rag_reduction=True, progress=progress, k=k, j=j, sample_budget=sample_budget, seed=seed, exp_name=exp_name, priors=priors, ) data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator) print(data_record_collection.to_df()) data_record_collection.to_df().to_csv(f"pareto-cascades-data/{exp_name}-output.csv", index=False) # create filepaths for records and stats records_path = f"pareto-cascades-data/{exp_name}-records.json" stats_path = f"pareto-cascades-data/{exp_name}-profiling.json" # save record outputs record_jsons = [] for record in data_record_collection: record_dict = record.to_dict() record_dict = { k: v for k, v in record_dict.items() if k in ["pmid", "reactions", "reaction_labels", "ranked_reaction_labels"] } record_jsons.append(record_dict) with open(records_path, "w") as f: json.dump(record_jsons, f) # save statistics execution_stats_dict = data_record_collection.execution_stats.to_json() with open(stats_path, "w") as f: json.dump(execution_stats_dict, f) # score output test_dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split="test").to_pandas() test_dataset = test_dataset.sample(n=250, random_state=seed).to_dict(orient="records") # construct mapping from pmid --> label (field, value) pairs def compute_target_record(entry): reactions_lst = [ reaction.strip().lower().replace("'", "").replace("^", "") for reaction in entry["reactions"].split(",") ] label_dict = {"ranked_reaction_labels": reactions_lst} return label_dict label_fields_to_values = { entry["pmid"]: compute_target_record(entry) for entry in test_dataset } def rank_precision_at_k(preds: list, targets: list, k: int): if preds is None: return 0.0 # lower-case each list preds = [pred.lower().replace("'", "").replace("^", "") for pred in preds] targets = set([target.lower().replace("'", "").replace("^", "") for target in targets]) # compute rank-precision at k rn = len(targets) denom = min(k, rn) total = 0.0 for i in range(k): total += preds[i] in targets if i < len(preds) else 0.0 return total / denom def compute_avg_rp_at_k(records, k=5): total_rp_at_k, bad = 0, 0 for record in records: pmid = record['pmid'] preds = record['ranked_reaction_labels'] targets = label_fields_to_values[pmid]['ranked_reaction_labels'] try: total_rp_at_k += rank_precision_at_k(preds, targets, k) except Exception: print(f"Error computing rank precision at k for record with pmid {pmid}") bad += 1 return total_rp_at_k / len(records), bad rp_at_k, failed = compute_avg_rp_at_k(record_jsons, k=5) final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0] final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id] stats_dict = { "rp@5": rp_at_k, "failed": failed, "optimization_time": data_record_collection.execution_stats.optimization_time, "optimization_cost": data_record_collection.execution_stats.optimization_cost, "plan_execution_time": data_record_collection.execution_stats.plan_execution_time, "plan_execution_cost": data_record_collection.execution_stats.plan_execution_cost, "total_execution_time": data_record_collection.execution_stats.total_execution_time, "total_execution_cost": data_record_collection.execution_stats.total_execution_cost, "plan_str": final_plan_str, } with open(f"pareto-cascades-data/{exp_name}-metrics.json", "w") as f: json.dump(stats_dict, f) print(f"rp@k: {rp_at_k:.5f}") print(f"failed: {failed}") print(f"Optimization time: {data_record_collection.execution_stats.optimization_time}") print(f"Optimization cost: {data_record_collection.execution_stats.optimization_cost}") print(f"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}") print(f"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}") print(f"Total Execution time: {data_record_collection.execution_stats.total_execution_time}") print(f"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}") ================================================ FILE: abacus-research/biodex-priors-cascades.json ================================================ {"0005c18b69": {"quality": 0.19444444444444442, "cost": 0.0038703809999999996, "time": 61.16110005378724}, "009df798a3": {"quality": 0.09252136752136753, "cost": 0.003526569, "time": 74.32940173149109}, "00c93aec22": {"quality": 0.10641025641025642, "cost": 0.010484405999999998, "time": 72.25670802593231}, "00e1fecc4c": {"quality": 0.21752136752136753, "cost": 0.006413474000000001, "time": 63.18833725452423}, "00f4acd0d3": {"quality": 0.0, "cost": 0.005918445, "time": 48.42981550693512}, "01413aa72d": {"quality": 0.16538461538461538, "cost": 0.008015024, "time": 69.29275906085968}, "01c2f973ad": {"quality": 0.1626068376068376, "cost": 0.002413815, "time": 42.94722969532013}, "02078988c1": {"quality": 0.0, "cost": 0.006904731999999999, "time": 27.019422554969786}, "021604dec1": {"quality": 0.3175213675213675, "cost": 0.008399656999999998, "time": 67.33289885520935}, "02410c662e": {"quality": 0.21752136752136753, "cost": 0.006360101999999999, "time": 77.91296794414521}, "0262668df7": {"quality": 0.21752136752136753, "cost": 0.0075904909999999996, "time": 59.82583842277527}, "0267c97b70": {"quality": 0.0, "cost": 0.003249014999999999, "time": 50.19215798377991}, "02ae38e4aa": {"quality": 0.1876068376068376, "cost": 0.004417125, "time": 69.9335319519043}, "02f49fe0fd": {"quality": 0.14807692307692308, "cost": 0.00044366999999999996, "time": 35.21025230884552}, "030756558c": {"quality": 0.04444444444444444, "cost": 0.002745915, "time": 53.435633063316345}, "033ca325e6": {"quality": 0.20277777777777778, "cost": 0.002408562, "time": 27.521768379211426}, "038a5f0a62": {"quality": 0.1876068376068376, "cost": 0.006526305, "time": 52.622441697120664}, "041b5af43d": {"quality": 0.21752136752136753, "cost": 0.010993874, "time": 64.76753988265992}, "042d933706": {"quality": 0.19444444444444442, "cost": 0.008023619, "time": 57.533840489387515}, "04397effa0": {"quality": 0.2064102564102564, "cost": 0.007009628999999999, "time": 76.10882368087769}, "0539e0b42d": {"quality": 0.25085470085470085, "cost": 0.007662647000000001, "time": 80.42491667270662}, "0554568b86": {"quality": 0.14038461538461539, "cost": 0.008002722, "time": 58.663866949081424}, "06493715cc": {"quality": 0.02222222222222222, "cost": 0.005358681, "time": 78.51178731918336}, "067ee6e91b": {"quality": 0.2098290598290598, "cost": 0.0072115949999999995, "time": 56.89639482498169}, "068b66f00d": {"quality": 0.1, "cost": 0.006528483999999999, "time": 68.53199887275696}, "0695f9b5fc": {"quality": 0.06944444444444445, "cost": 0.007583937, "time": 73.26675381660462}, "073ed5b301": {"quality": 0.07371794871794872, "cost": 0.005419403999999999, "time": 60.23991825580597}, "079feb14a8": {"quality": 0.2098290598290598, "cost": 0.008877133, "time": 62.487732887268066}, "07a3a7daf7": {"quality": 0.0, "cost": 0.006907794, "time": 43.434891891479495}, "08127cd6dd": {"quality": 0.1987179487179487, "cost": 0.012973673000000002, "time": 55.40713529586792}, "0833133620": {"quality": 0.1987179487179487, "cost": 0.005889005999999999, "time": 65.5436268568039}, "089565077c": {"quality": 0.025, "cost": 0.000339832, "time": 65.31151757240295}, "08bf8cc191": {"quality": 0.10555555555555557, "cost": 0.011634817000000002, "time": 95.24387204647064}, "08e1802287": {"quality": 0.23974358974358972, "cost": 0.0033798820000000003, "time": 25.597942876815797}, "090cd3ef31": {"quality": 0.23974358974358972, "cost": 0.0031741180000000005, "time": 25.463910150527955}, "0944a921e8": {"quality": 0.0702991452991453, "cost": 0.00792617, "time": 85.74197387695312}, "0947216ece": {"quality": 0.1987179487179487, "cost": 0.015770336, "time": 82.8155886888504}, "096d51f670": {"quality": 0.18482905982905984, "cost": 0.009268239, "time": 104.61172659397124}, "09791c731b": {"quality": 0.0438034188034188, "cost": 0.006413886, "time": 85.14133274555206}, "0990c0d4f8": {"quality": 0.2098290598290598, "cost": 0.007531591, "time": 77.474085521698}, "0a128688c1": {"quality": 0.18333333333333332, "cost": 0.014743780000000001, "time": 100.52884330749512}, "0a4c1bbb4a": {"quality": 0.16538461538461538, "cost": 0.005475987, "time": 77.0306186914444}, "0ac969dde3": {"quality": 0.19444444444444442, "cost": 0.009684573, "time": 102.05468263626099}, "0af1efab0e": {"quality": 0.0, "cost": 0.0035705999999999993, "time": 76.96206550598144}, "0b1ed7ff58": {"quality": 0.19252136752136753, "cost": 0.008049169, "time": 92.94120838642121}, "0b3dc2e896": {"quality": 0.0, "cost": 0.00892118, "time": 96.49321339130401}, "0b43e94f3f": {"quality": 0.1987179487179487, "cost": 0.001608864, "time": 40.973159313201904}, "0b4ab72197": {"quality": 0.061111111111111116, "cost": 0.010094982, "time": 72.49727232456206}, "0be862a0dc": {"quality": 0.04038461538461539, "cost": 0.0062549730000000005, "time": 56.527322101593015}, "0bf3129ae8": {"quality": 0.16538461538461538, "cost": 0.0028206660000000003, "time": 54.240378093719485}, "0c020b86a3": {"quality": 0.18482905982905984, "cost": 0.008657178000000001, "time": 49.74211900234222}, "0c6c7fe96a": {"quality": 0.12863247863247865, "cost": 0.012195626999999999, "time": 80.83456852436066}, "0c81c8996a": {"quality": 0.023076923076923078, "cost": 0.006098082000000001, "time": 71.85198886394501}, "0cdc5954dd": {"quality": 0.11666666666666667, "cost": 0.008310576, "time": 66.0079866170883}, "0d25188bf7": {"quality": 0.04807692307692308, "cost": 0.008176553999999999, "time": 84.21552875041962}, "0d9d767ae5": {"quality": 0.04444444444444444, "cost": 0.012340264, "time": 71.0254203081131}, "0e36342fe7": {"quality": 0.09252136752136753, "cost": 0.00467736, "time": 38.08519749641418}, "0e7e862290": {"quality": 0.0, "cost": 0.003672576, "time": 74.36907055377961}, "0e91cd07f9": {"quality": 0.18333333333333332, "cost": 0.006350159999999999, "time": 76.38928816318511}, "0ec672e7c8": {"quality": 0.2098290598290598, "cost": 0.013312296999999999, "time": 90.61400089263915}, "0ed243f788": {"quality": 0.23205128205128206, "cost": 0.008812347, "time": 80.54674921035766}, "0eeb372802": {"quality": 0.23974358974358972, "cost": 0.013314724, "time": 94.53299105167389}, "0effe9b1dc": {"quality": 0.15918803418803418, "cost": 0.003532296, "time": 72.70150463581085}, "0f7faf684d": {"quality": 0.10982905982905983, "cost": 0.011855149000000002, "time": 89.06640274524689}, "0fcec544e3": {"quality": 0.25982905982905985, "cost": 0.004687262999999999, "time": 83.44144034385681}, "0ff126ebf8": {"quality": 0.21752136752136753, "cost": 0.011823445, "time": 64.27427606582641}, "112d9a3421": {"quality": 0.2730769230769231, "cost": 0.010725950000000001, "time": 87.46276273727418}, "114a097c53": {"quality": 0.10982905982905983, "cost": 0.00542718, "time": 47.13481698036194}, "116334cd72": {"quality": 0.3175213675213675, "cost": 0.012438881999999998, "time": 93.07386040687561}, "1175ee37e6": {"quality": 0.19444444444444442, "cost": 0.001459101, "time": 30.085742378234862}, "11a66478dc": {"quality": 0.125, "cost": 0.00746376, "time": 63.90720744132996}, "11bc996d48": {"quality": 0.2098290598290598, "cost": 0.0065042699999999995, "time": 65.52873225212097}, "11debf9fc0": {"quality": 0.18482905982905984, "cost": 0.006505245, "time": 89.37637939453126}, "123fb650fb": {"quality": 0.21752136752136753, "cost": 0.008327412, "time": 88.56314113140107}, "1274c21076": {"quality": 0.125, "cost": 0.011184342, "time": 102.6374900341034}, "127af50739": {"quality": 0.16538461538461538, "cost": 0.008449134, "time": 104.71919329166413}, "12addbf5e2": {"quality": 0.04444444444444444, "cost": 0.003276834, "time": 88.02170798778533}, "133ee5023f": {"quality": 0.1987179487179487, "cost": 0.005588716, "time": 53.45017175674438}, "1368e1c78e": {"quality": 0.16752136752136754, "cost": 0.0034759379999999996, "time": 84.43210818767548}, "13a009fe0c": {"quality": 0.21752136752136753, "cost": 0.012945674, "time": 98.5892866373062}, "13da306f84": {"quality": 0.10363247863247865, "cost": 0.0087151, "time": 77.5059399843216}, "13f2b9c25b": {"quality": 0.25982905982905985, "cost": 0.003494103, "time": 98.82596600055695}, "13f75f9bd0": {"quality": 0.2064102564102564, "cost": 0.007151064, "time": 70.01372668743133}, "1404e0aa35": {"quality": 0.2098290598290598, "cost": 0.007724231, "time": 94.99440484046936}, "140ededb41": {"quality": 0.2098290598290598, "cost": 0.001514445, "time": 19.843628692626954}, "142f3a7c70": {"quality": 0.0, "cost": 0.004404129, "time": 77.21951529979705}, "1468dddecc": {"quality": 0.16944444444444445, "cost": 0.002386455, "time": 48.906515979766844}, "14d19a01e2": {"quality": 0.05641025641025641, "cost": 0.0063798959999999995, "time": 72.75729236602783}, "1624bb5302": {"quality": 0.18482905982905984, "cost": 0.001919664, "time": 66.44049925804138}, "1636e0833b": {"quality": 0.025, "cost": 0.002752764, "time": 74.27101521492004}, "1658296f3a": {"quality": 0.21752136752136753, "cost": 0.005375874, "time": 64.17133255004883}, "16f351273f": {"quality": 0.17307692307692307, "cost": 0.01028627, "time": 67.0687807559967}, "171e6ae293": {"quality": 0.21752136752136753, "cost": 0.007912404000000001, "time": 62.13353226184845}, "176da24f53": {"quality": 0.015384615384615385, "cost": 0.0038477320000000004, "time": 42.74407241344451}, "179379555f": {"quality": 0.1987179487179487, "cost": 0.006129615, "time": 88.43889818191528}, "17c928174f": {"quality": 0.1987179487179487, "cost": 0.0028421130000000003, "time": 75.59559867382049}, "181c91d1be": {"quality": 0.21752136752136753, "cost": 0.012095439999999999, "time": 67.77491972446441}, "18368684cd": {"quality": 0.05641025641025641, "cost": 0.0059048880000000005, "time": 90.52595224380494}, "183743e76e": {"quality": 0.0, "cost": 0.005899368, "time": 74.72685542106629}, "18f55750b0": {"quality": 0.0, "cost": 0.008405136, "time": 124.15524208545685}, "19563b057d": {"quality": 0.1987179487179487, "cost": 0.011058744, "time": 94.49535017013551}, "1957127275": {"quality": 0.29252136752136754, "cost": 0.008193474000000001, "time": 92.91692547798156}, "197bb53f10": {"quality": 0.2098290598290598, "cost": 0.006091916999999999, "time": 79.83643505573272}, "199fd1fbf2": {"quality": 0.058333333333333334, "cost": 0.00121965, "time": 48.699701046943666}, "19b40e0271": {"quality": 0.3175213675213675, "cost": 0.008674995000000001, "time": 116.94037146568297}, "19e3db7fe7": {"quality": 0.025, "cost": 0.01204651, "time": 117.68800120353698}, "1ad856985f": {"quality": 0.19444444444444442, "cost": 0.008808637000000001, "time": 115.93177556991577}, "1adec2dca2": {"quality": 0.0, "cost": 0.013760218000000001, "time": 125.3757052898407}, "1b04a2b184": {"quality": 0.09252136752136753, "cost": 0.013170014, "time": 111.55746636390685}, "1b28439bd7": {"quality": 0.21752136752136753, "cost": 0.010848586, "time": 84.68317291736602}, "1beb2fac62": {"quality": 0.3175213675213675, "cost": 0.013538090000000001, "time": 73.24731080532074}, "1c347e4d91": {"quality": 0.11752136752136753, "cost": 0.003958569, "time": 81.87744581699371}, "1c3882926e": {"quality": 0.06538461538461539, "cost": 0.006650534999999999, "time": 102.02758514881134}, "1cc6d9efb6": {"quality": 0.21752136752136753, "cost": 0.011642267999999999, "time": 73.37450892925263}, "1ce3d77039": {"quality": 0.1987179487179487, "cost": 0.000591721, "time": 33.82067358493805}, "1ce99cf2c8": {"quality": 0.19252136752136753, "cost": 0.007676729, "time": 109.94239032268524}, "1d26090364": {"quality": 0.08141025641025641, "cost": 0.004802969999999999, "time": 37.3655684709549}, "1d87f97e62": {"quality": 0.25085470085470085, "cost": 0.0074821539999999995, "time": 69.90373904705048}, "1da2369719": {"quality": 0.1737179487179487, "cost": 0.003804339, "time": 75.42430078983307}, "1e18e60895": {"quality": 0.1952991452991453, "cost": 0.000637105, "time": 36.58538358211517}, "1e1bf7e88b": {"quality": 0.2064102564102564, "cost": 0.012300497, "time": 91.10763094425201}, "1e8b3521f8": {"quality": 0.23205128205128206, "cost": 0.003055887, "time": 85.00319654941559}, "1f5e8c9e9a": {"quality": 0.06538461538461539, "cost": 0.01254384, "time": 110.50039112567902}, "2018bef45f": {"quality": 0.13974358974358975, "cost": 0.0073448359999999996, "time": 90.09606876373292}, "2066966577": {"quality": 0.19444444444444442, "cost": 0.011873376, "time": 119.55739791393279}, "2075ff1d04": {"quality": 0.08333333333333334, "cost": 0.0009685659999999999, "time": 70.29870116710663}, "2080b60a57": {"quality": 0.04444444444444444, "cost": 0.005072787, "time": 116.32027242183685}, "208a98f514": {"quality": 0.21752136752136753, "cost": 0.009998995, "time": 118.32991974353791}, "20e10af7d4": {"quality": 0.2098290598290598, "cost": 0.005652002999999999, "time": 116.42957108020784}, "20e2c0b057": {"quality": 0.10982905982905983, "cost": 0.007542789, "time": 113.53230805397033}, "211b89b4cd": {"quality": 0.1952991452991453, "cost": 0.005615415, "time": 63.006171917915346}, "2153174e2d": {"quality": 0.11474358974358972, "cost": 0.007281448, "time": 118.94957902431489}, "21b2b8ebd1": {"quality": 0.21752136752136753, "cost": 0.007063268000000001, "time": 76.91416993141175}, "21b2df8512": {"quality": 0.19444444444444442, "cost": 0.00326791, "time": 69.47645666599274}, "21bed16a7d": {"quality": 0.07307692307692308, "cost": 0.0016491099999999999, "time": 30.10742793083191}, "227246dff8": {"quality": 0.1876068376068376, "cost": 0.0006464099999999999, "time": 42.72886557579041}, "227c30d349": {"quality": 0.1, "cost": 0.005918874, "time": 103.58872454166413}, "228687831a": {"quality": 0.04722222222222222, "cost": 0.005076029999999999, "time": 67.8051172733307}, "23075b2a6e": {"quality": 0.05, "cost": 0.0024716549999999997, "time": 97.33722229003905}, "23566f15ab": {"quality": 0.1876068376068376, "cost": 0.00044987499999999997, "time": 40.82834756374359}, "2370cebb10": {"quality": 0.21752136752136753, "cost": 0.004052682, "time": 106.21984114646912}, "2386b03c4c": {"quality": 0.1987179487179487, "cost": 0.006683786999999999, "time": 110.57667918205262}, "24957f3a43": {"quality": 0.1987179487179487, "cost": 0.004216121999999999, "time": 70.75154151916504}, "24c122de4e": {"quality": 0.061111111111111116, "cost": 0.0059287260000000005, "time": 78.56412732601166}, "24f76747b9": {"quality": 0.14871794871794872, "cost": 0.003334974, "time": 96.20667114257813}, "2609bfd616": {"quality": 0.08482905982905983, "cost": 0.009335312, "time": 101.31035015583038}, "260ab3e966": {"quality": 0.18482905982905984, "cost": 0.006512652000000001, "time": 107.31265118122101}, "2728c8eb6a": {"quality": 0.05918803418803419, "cost": 0.009020646, "time": 82.33452692031861}, "27971eaaf5": {"quality": 0.1876068376068376, "cost": 0.005893562, "time": 85.50352709293365}, "27ba0964b2": {"quality": 0.18205128205128207, "cost": 0.0012502439999999997, "time": 50.85352149009705}, "27daa50458": {"quality": 0.21752136752136753, "cost": 0.005911454, "time": 49.84150557518005}, "28369b2421": {"quality": 0.3175213675213675, "cost": 0.008581950000000001, "time": 82.63782026767731}, "28421e6d62": {"quality": 0.05, "cost": 0.0071309500000000005, "time": 77.62054443359375}, "2848c42f91": {"quality": 0.0, "cost": 0.004609089, "time": 92.82525777816772}, "28a638bb6e": {"quality": 0.04444444444444444, "cost": 0.0037765230000000004, "time": 103.16399810314178}, "290947fe5a": {"quality": 0.08482905982905983, "cost": 0.005273313, "time": 115.58293502330781}, "2936c3e43e": {"quality": 0.13205128205128205, "cost": 0.011067623, "time": 92.91213045120239}, "293ec5edca": {"quality": 0.2987179487179487, "cost": 0.006866393, "time": 77.98297226428986}, "294e541235": {"quality": 0.0, "cost": 0.0024907139999999998, "time": 55.42697856426239}, "295ed5e759": {"quality": 0.023076923076923078, "cost": 0.007770515999999999, "time": 73.042240858078}, "2960431101": {"quality": 0.21752136752136753, "cost": 0.007103678, "time": 72.23796293735504}, "29892d8468": {"quality": 0.0, "cost": 0.011991246, "time": 90.30782704353332}, "299a0aeb65": {"quality": 0.21752136752136753, "cost": 0.008716849999999998, "time": 87.09911065101625}, "29bf3c0a3b": {"quality": 0.0952991452991453, "cost": 0.008317653000000001, "time": 95.62355568408967}, "29c8c693e2": {"quality": 0.19252136752136753, "cost": 0.009049301000000001, "time": 97.66356644630432}, "2a7d15f4a7": {"quality": 0.1987179487179487, "cost": 0.0007805049999999999, "time": 26.81472017765045}, "2ae24e0124": {"quality": 0.2098290598290598, "cost": 0.006997367999999999, "time": 100.80345997810363}, "2b5679d248": {"quality": 0.23482905982905983, "cost": 0.012146391000000001, "time": 73.05133624076844}, "2b5ab72a55": {"quality": 0.1, "cost": 0.00048774399999999997, "time": 87.95751609802247}, "2b82a67eb1": {"quality": 0.2098290598290598, "cost": 0.002022681, "time": 73.1611754655838}, "2bcbffdf85": {"quality": 0.1, "cost": 0.003530234999999999, "time": 89.18944058418273}, "2bd39ee744": {"quality": 0.2098290598290598, "cost": 0.014582621000000002, "time": 59.03191387653351}, "2bf38d797f": {"quality": 0.2064102564102564, "cost": 0.0075413270000000004, "time": 54.856262516975406}, "2c1640adf7": {"quality": 0.18482905982905984, "cost": 0.0019162139999999999, "time": 77.58103239536285}, "2c5cf9eb26": {"quality": 0.125, "cost": 0.006593481, "time": 71.76690304279327}, "2c87313a93": {"quality": 0.11752136752136753, "cost": 0.012109324000000001, "time": 136.4559998512268}, "2c9a9f94c4": {"quality": 0.06538461538461539, "cost": 0.011352702, "time": 108.1334749698639}, "2d3bbc2d23": {"quality": 0.1987179487179487, "cost": 0.006806840999999999, "time": 94.72236230373383}, "2d7f1dbd4b": {"quality": 0.1987179487179487, "cost": 0.003597009, "time": 112.8651005268097}, "2de113167b": {"quality": 0.11752136752136753, "cost": 0.011756008, "time": 101.33509397506714}, "2de3eb2c19": {"quality": 0.23974358974358972, "cost": 0.006698560000000001, "time": 58.83604803085328}, "2e02b71061": {"quality": 0.09444444444444444, "cost": 0.002093574, "time": 88.67284088134765}, "2e30394ac6": {"quality": 0.07307692307692308, "cost": 0.007492572, "time": 90.685981965065}, "2e5d071f21": {"quality": 0.12863247863247865, "cost": 0.0072971500000000005, "time": 123.94845788478851}, "2e9c5cc9bf": {"quality": 0.05, "cost": 0.005878296000000001, "time": 49.976054430007935}, "2ec4bec1a3": {"quality": 0.21474358974358973, "cost": 0.010779144000000001, "time": 124.55905365943909}, "2f1573da80": {"quality": 0.1987179487179487, "cost": 0.008802546, "time": 103.0501886844635}, "2fc0cb3592": {"quality": 0.06538461538461539, "cost": 0.00690309, "time": 73.22720482349396}, "2fd9cd426a": {"quality": 0.21752136752136753, "cost": 0.01168472, "time": 97.46322917938232}, "3019af79b3": {"quality": 0.1, "cost": 0.001074796, "time": 51.414027357101446}, "302c1d97fc": {"quality": 0.1814102564102564, "cost": 0.01244514, "time": 70.44251236915588}, "303b467574": {"quality": 0.21752136752136753, "cost": 0.007695415999999999, "time": 104.79880075454712}, "3058b1f1f8": {"quality": 0.12863247863247865, "cost": 0.006238776, "time": 104.19706840515136}, "30ae4cbe91": {"quality": 0.19252136752136753, "cost": 0.000758614, "time": 64.8543310880661}, "30e3ff1d17": {"quality": 0.1841880341880342, "cost": 0.007638456, "time": 97.64294464588164}, "30f20c8fe6": {"quality": 0.1876068376068376, "cost": 0.006185881000000001, "time": 95.83135304450988}, "316759d191": {"quality": 0.06538461538461539, "cost": 0.01277288, "time": 106.51430850028991}, "3177802176": {"quality": 0.05, "cost": 0.000812176, "time": 68.97149279117585}, "3184f977a8": {"quality": 0.06944444444444445, "cost": 0.003731166, "time": 116.78110229969025}, "3194e440cf": {"quality": 0.1, "cost": 0.0021085889999999997, "time": 68.94484202861786}, "3197ad4faf": {"quality": 0.125, "cost": 0.0029357279999999994, "time": 104.7419487476349}, "31a423a3bf": {"quality": 0.0, "cost": 0.002416362, "time": 71.78247156143189}, "321e17afbd": {"quality": 0.1952991452991453, "cost": 0.009030642000000002, "time": 102.16506278514862}, "32b101d807": {"quality": 0.21752136752136753, "cost": 0.013004699000000002, "time": 107.63277230262756}, "332a350ea2": {"quality": 0.10363247863247865, "cost": 0.005886114, "time": 79.25630362033844}, "33459cd29c": {"quality": 0.02222222222222222, "cost": 0.009792765999999998, "time": 109.5698808670044}, "33a187e74f": {"quality": 0.1987179487179487, "cost": 0.0057371969999999994, "time": 108.5205159664154}, "34026bb5cc": {"quality": 0.2098290598290598, "cost": 0.003850629, "time": 79.99503300189971}, "3511b5e1d0": {"quality": 0.0, "cost": 0.007917534, "time": 106.0169960975647}, "3513e54767": {"quality": 0.12307692307692308, "cost": 0.00652464, "time": 56.468523144721985}, "353f0cb1ac": {"quality": 0.2064102564102564, "cost": 0.0027354140000000002, "time": 50.53772373199463}, "3550bf88cb": {"quality": 0.22863247863247865, "cost": 0.007747509, "time": 104.70380001068116}, "35610fb420": {"quality": 0.2098290598290598, "cost": 0.004438119, "time": 84.96952936649322}, "357267e14b": {"quality": 0.17222222222222222, "cost": 0.00107605, "time": 39.50561866760254}, "36011c7606": {"quality": 0.1987179487179487, "cost": 0.008097354, "time": 84.37511310577392}, "362d480d6d": {"quality": 0.22863247863247865, "cost": 0.010916724, "time": 110.61522128582001}, "363209b6e7": {"quality": 0.32863247863247863, "cost": 0.006868834000000001, "time": 83.50905873775483}, "3637084f91": {"quality": 0.3175213675213675, "cost": 0.007119692, "time": 82.10715711116791}, "36b17c40f3": {"quality": 0.03418803418803419, "cost": 0.004674881999999999, "time": 53.087517738342285}, "36c66671ee": {"quality": 0.12307692307692308, "cost": 0.006580928999999999, "time": 86.52281455993652}, "372e8b5f4f": {"quality": 0.058333333333333334, "cost": 0.001610232, "time": 70.54334411621093}, "375ed248fe": {"quality": 0.06752136752136753, "cost": 0.008873826000000001, "time": 86.2198546409607}, "377b8b0bcc": {"quality": 0.025, "cost": 0.0049981109999999995, "time": 106.38127601146698}, "37bd28f2c9": {"quality": 0.21752136752136753, "cost": 0.007961847, "time": 103.88883001804352}, "38075bb01f": {"quality": 0.21666666666666667, "cost": 0.003875973, "time": 75.78762099742889}, "38567d6a43": {"quality": 0.0, "cost": 0.004616319, "time": 89.08885374069214}, "389a99ab21": {"quality": 0.07371794871794872, "cost": 0.007963053, "time": 102.66287496089936}, "389c54cbca": {"quality": 0.0702991452991453, "cost": 0.005610244, "time": 48.15491397380829}, "38ec11cf7b": {"quality": 0.0, "cost": 0.007187318999999999, "time": 103.98532650470733}, "3980f20caa": {"quality": 0.04807692307692308, "cost": 0.014391898, "time": 100.08462851047516}, "39cd4ca402": {"quality": 0.025, "cost": 0.003120591, "time": 75.18385965824127}, "3a34b24c41": {"quality": 0.21752136752136753, "cost": 0.002965395, "time": 73.19122822284699}, "3b2e8075ea": {"quality": 0.2098290598290598, "cost": 0.003075393, "time": 70.68897068500519}, "3b3676521a": {"quality": 0.21752136752136753, "cost": 0.013715839000000002, "time": 107.15725564956665}, "3b3a6bf087": {"quality": 0.30982905982905984, "cost": 0.006470724000000001, "time": 74.07467935085296}, "3b4bde0121": {"quality": 0.0, "cost": 0.00302925, "time": 77.53029568195343}, "3b57530a56": {"quality": 0.2098290598290598, "cost": 0.002663507, "time": 48.332898473739625}, "3c206c89f3": {"quality": 0.21752136752136753, "cost": 0.008676515999999999, "time": 115.085169839859}, "3cbab8082e": {"quality": 0.09871794871794873, "cost": 0.008985135, "time": 116.89071977138519}, "3d71c4dd2c": {"quality": 0.10982905982905983, "cost": 0.006464912999999999, "time": 86.58996284008026}, "3ea15ac20c": {"quality": 0.11538461538461539, "cost": 0.007241556, "time": 125.68785231113434}, "3f1a58aec9": {"quality": 0.22863247863247865, "cost": 0.00189393, "time": 45.61551144123077}, "3f2321bb08": {"quality": 0.125, "cost": 0.000410382, "time": 33.271996068954465}, "3f3ef494b0": {"quality": 0.0, "cost": 0.001176912, "time": 50.03896124362946}, "3f62c3fbfc": {"quality": 0.21752136752136753, "cost": 0.004284207, "time": 95.22124736309053}, "3f88dd99f7": {"quality": 0.19252136752136753, "cost": 0.005681200000000001, "time": 63.5021466255188}, "3fa747af9a": {"quality": 0.15833333333333333, "cost": 0.002231574, "time": 96.99003422260284}, "40104c813f": {"quality": 0.0, "cost": 0.010208657000000001, "time": 125.99128649234771}, "403b05da2d": {"quality": 0.08141025641025641, "cost": 0.010538071999999999, "time": 94.6719701051712}, "4043815a3e": {"quality": 0.21752136752136753, "cost": 0.013145456, "time": 124.91901173591614}, "409ff67607": {"quality": 0.06944444444444445, "cost": 0.004053711, "time": 99.38009803295135}, "412c065b83": {"quality": 0.0, "cost": 0.008236578, "time": 92.44097802639007}, "4171fbac5c": {"quality": 0.19444444444444442, "cost": 0.009067686, "time": 114.00634713172913}, "4191118787": {"quality": 0.1987179487179487, "cost": 0.008734028999999999, "time": 109.63090167045593}, "41d5b97871": {"quality": 0.0, "cost": 0.002514906, "time": 62.46952087879181}, "41d8845655": {"quality": 0.2098290598290598, "cost": 0.010189181000000002, "time": 59.657727408409116}, "41ee202cac": {"quality": 0.06538461538461539, "cost": 0.000921442, "time": 54.925141382217404}, "42082dcd0d": {"quality": 0.1987179487179487, "cost": 0.007583567999999999, "time": 108.27059605121613}, "42ddd48341": {"quality": 0.22094017094017093, "cost": 0.014396616, "time": 104.56641755104064}, "4361bc7ea7": {"quality": 0.13333333333333333, "cost": 0.001473496, "time": 63.439108324050906}, "43afdad250": {"quality": 0.22863247863247865, "cost": 0.011869643, "time": 108.28341348171234}, "43c3cf9cb8": {"quality": 0.2064102564102564, "cost": 0.0068331120000000006, "time": 53.15135598182678}, "43d24fb32a": {"quality": 0.07307692307692308, "cost": 0.00976587, "time": 85.51874697208405}, "440bc872de": {"quality": 0.07222222222222222, "cost": 0.007188219, "time": 121.43468098640442}, "44173a9aef": {"quality": 0.30641025641025643, "cost": 0.01244934, "time": 115.21545617580415}, "44d6af5523": {"quality": 0.2098290598290598, "cost": 0.00476793, "time": 81.34744908809662}, "44fe4e4e3e": {"quality": 0.11752136752136753, "cost": 0.01136529, "time": 81.14654626846314}, "4587a1500c": {"quality": 0.3175213675213675, "cost": 0.006090974000000002, "time": 79.77526035308838}, "45ef93b61e": {"quality": 0.1987179487179487, "cost": 0.003150051, "time": 81.59605078697206}, "461846a52d": {"quality": 0.1814102564102564, "cost": 0.003777498, "time": 82.82821555137635}, "462e6ff849": {"quality": 0.11752136752136753, "cost": 0.010684958, "time": 113.33498125076294}, "4630853d32": {"quality": 0.06752136752136753, "cost": 0.008278823000000001, "time": 86.25940473079682}, "46475b9e75": {"quality": 0.2098290598290598, "cost": 0.0061740449999999995, "time": 87.75788187980652}, "46654a1f32": {"quality": 0.0, "cost": 0.011978384, "time": 67.87502360343933}, "466a3036b2": {"quality": 0.09252136752136753, "cost": 0.010652967, "time": 114.7115253686905}, "466d4d16dd": {"quality": 0.18482905982905984, "cost": 0.008396510000000001, "time": 86.67418849468231}, "46a35022d8": {"quality": 0.1952991452991453, "cost": 0.000766365, "time": 59.83990566730499}, "46ed68152d": {"quality": 0.18333333333333332, "cost": 0.01143953, "time": 92.87648718357087}, "46edc488a4": {"quality": 0.059829059829059825, "cost": 0.0012849899999999997, "time": 50.55697724819183}, "476a12876c": {"quality": 0.11752136752136753, "cost": 0.006154817999999999, "time": 81.70997877120972}, "48043e2304": {"quality": 0.2098290598290598, "cost": 0.009217511000000001, "time": 89.0722332715988}, "488645cbd9": {"quality": 0.2064102564102564, "cost": 0.001493856, "time": 48.69374096393585}, "49009a3b57": {"quality": 0.059829059829059825, "cost": 0.007598573999999999, "time": 78.47919590473174}, "4909061216": {"quality": 0.21752136752136753, "cost": 0.013011773, "time": 104.75487668514252}, "49107972df": {"quality": 0.025, "cost": 0.007639823999999999, "time": 118.11295173168182}, "49731b1ccd": {"quality": 0.0952991452991453, "cost": 0.006953804999999999, "time": 77.60816009044646}, "498f146004": {"quality": 0.11752136752136753, "cost": 0.008050870000000002, "time": 112.59319038391114}, "49ad844bd2": {"quality": 0.2098290598290598, "cost": 0.008512473, "time": 109.38159234523773}, "49ca727e49": {"quality": 0.2098290598290598, "cost": 0.000746866, "time": 34.221199584007266}, "4a4a960a82": {"quality": 0.13333333333333333, "cost": 0.007445242000000001, "time": 84.403648686409}, "4a92372986": {"quality": 0.16538461538461538, "cost": 0.00319335, "time": 83.45380291938781}, "4aa7e8fde6": {"quality": 0.10641025641025642, "cost": 0.0019865879999999996, "time": 83.08999395370483}, "4aafd39d76": {"quality": 0.14038461538461539, "cost": 0.005373513, "time": 82.9023279428482}, "4ace1cfad1": {"quality": 0.16538461538461538, "cost": 0.009508553999999999, "time": 106.34385075569153}, "4ad1952206": {"quality": 0.20213675213675214, "cost": 0.0085493, "time": 83.97655324935913}, "4b59f40131": {"quality": 0.04807692307692308, "cost": 0.006207440999999999, "time": 83.43397681713105}, "4b86a1c038": {"quality": 0.2064102564102564, "cost": 0.00792555, "time": 106.1481963634491}, "4b92a26754": {"quality": 0.0, "cost": 0.00047688399999999996, "time": 69.90886387825012}, "4bc4528402": {"quality": 0.16538461538461538, "cost": 0.004527411, "time": 81.34518160820008}, "4c158a1a4a": {"quality": 0.08205128205128205, "cost": 0.007535195999999999, "time": 78.38852503299714}, "4c954323e3": {"quality": 0.21752136752136753, "cost": 0.00974773, "time": 103.17564594745636}, "4d8bcf8ae2": {"quality": 0.125, "cost": 0.000539128, "time": 69.75875072479249}, "4d91e8a27b": {"quality": 0.21752136752136753, "cost": 0.010584353000000001, "time": 76.16667878627777}, "4dd3635bc3": {"quality": 0.2098290598290598, "cost": 0.012259316000000001, "time": 78.35238783359529}, "4dd96bd18f": {"quality": 0.21752136752136753, "cost": 0.011986314000000001, "time": 105.92028946876526}, "4e298ee0d4": {"quality": 0.1814102564102564, "cost": 0.008259033, "time": 95.37437601089476}, "4e3443a0f9": {"quality": 0.11752136752136753, "cost": 0.013054975, "time": 106.09084448814392}, "4e4b9db2b8": {"quality": 0.21752136752136753, "cost": 0.007140224000000001, "time": 72.97610201835633}, "4e6509f614": {"quality": 0.061111111111111116, "cost": 0.0053766, "time": 47.53030514717102}, "4e6a83e751": {"quality": 0.07371794871794872, "cost": 0.001089142, "time": 53.57680480480194}, "4e8d8e527a": {"quality": 0.14871794871794872, "cost": 0.0051247079999999995, "time": 85.9967652797699}, "4ef333ab21": {"quality": 0.19444444444444442, "cost": 0.001917522, "time": 79.47004499435425}, "4f16545711": {"quality": 0.06666666666666667, "cost": 0.005466131999999999, "time": 88.67296268939972}, "4f8cca1195": {"quality": 0.21752136752136753, "cost": 0.010168095, "time": 114.15254590511321}, "500860eaa2": {"quality": 0.0, "cost": 0.0022933619999999997, "time": 38.243334197998045}, "50701b505e": {"quality": 0.1987179487179487, "cost": 0.0064327049999999995, "time": 79.18819501399994}, "50bc87e9cc": {"quality": 0.1987179487179487, "cost": 0.005166006000000001, "time": 86.32211146354675}, "50c03be77c": {"quality": 0.10982905982905983, "cost": 0.00168543, "time": 94.39419853687286}, "510375edad": {"quality": 0.0, "cost": 0.0021525, "time": 67.02195911407472}, "512fdb607c": {"quality": 0.2098290598290598, "cost": 0.008402651, "time": 120.24980294704437}, "51583a901c": {"quality": 0.2098290598290598, "cost": 0.010226283000000001, "time": 109.70203943252564}, "521314dab6": {"quality": 0.19252136752136753, "cost": 0.0064746199999999995, "time": 59.58354845046997}, "5241bf401b": {"quality": 0.07222222222222222, "cost": 0.006321017999999999, "time": 82.37689163684846}, "526878b5eb": {"quality": 0.07371794871794872, "cost": 0.008675337, "time": 115.97295072078705}, "52c1cba6ce": {"quality": 0.21752136752136753, "cost": 0.009671612999999999, "time": 108.82956576347351}, "52e5d0f4fb": {"quality": 0.2064102564102564, "cost": 0.001697778, "time": 88.22295272350311}, "52f041a70e": {"quality": 0.2098290598290598, "cost": 0.001067476, "time": 32.61202094554901}, "5307496302": {"quality": 0.2098290598290598, "cost": 0.005388657, "time": 86.62635931968688}, "53869388bb": {"quality": 0.061111111111111116, "cost": 0.0021395499999999996, "time": 39.578048658370975}, "53d2932c4f": {"quality": 0.2814102564102564, "cost": 0.007557482000000001, "time": 73.93080537319183}, "5474247f91": {"quality": 0.0, "cost": 0.008248308, "time": 80.75084838867187}, "557d2cf7ba": {"quality": 0.19444444444444442, "cost": 0.009174846, "time": 81.92483322620392}, "559c7120c5": {"quality": 0.14038461538461539, "cost": 0.009353153, "time": 113.30842261314393}, "55c8aa8935": {"quality": 0.04807692307692308, "cost": 0.01165586, "time": 114.18454928398131}, "56a29a28c5": {"quality": 0.04807692307692308, "cost": 0.004978499999999999, "time": 55.467578983306886}, "56b39eb1d6": {"quality": 0.21752136752136753, "cost": 0.009558101, "time": 131.33570635318756}, "5703697dbd": {"quality": 0.2098290598290598, "cost": 0.006108023999999999, "time": 79.37135038375854}, "5718f2ed80": {"quality": 0.10641025641025642, "cost": 0.003117972, "time": 91.63201060295106}, "572a02a59a": {"quality": 0.19444444444444442, "cost": 0.004321886999999999, "time": 100.45879077911377}, "572c2df793": {"quality": 0.04038461538461539, "cost": 0.004394319000000001, "time": 143.467453289032}, "5750713a41": {"quality": 0.2098290598290598, "cost": 0.006046722, "time": 92.31352968215941}, "57757ef15e": {"quality": 0.1876068376068376, "cost": 0.006326642, "time": 106.06430275440215}, "5793d14bbe": {"quality": 0.2098290598290598, "cost": 0.009510144000000002, "time": 139.83697164058685}, "579a915ed2": {"quality": 0.08333333333333334, "cost": 0.011355894000000002, "time": 109.32498943805695}, "579c81bbe0": {"quality": 0.025, "cost": 0.004824098, "time": 75.78087794780731}, "57bed1722f": {"quality": 0.025, "cost": 0.005056998, "time": 85.83169932365418}, "585ba6d20b": {"quality": 0.21752136752136753, "cost": 0.007866175, "time": 151.51093373298644}, "589a1cea79": {"quality": 0.05, "cost": 0.009551150000000001, "time": 151.7195835828781}, "59006532b4": {"quality": 0.16538461538461538, "cost": 0.007208474999999999, "time": 118.06772444248199}, "59326c4e00": {"quality": 0.015384615384615385, "cost": 0.005300058, "time": 72.70066883563996}, "596f0ed542": {"quality": 0.023076923076923078, "cost": 0.011558136, "time": 137.05188086032868}, "5971ba4e0d": {"quality": 0.21752136752136753, "cost": 0.0018305540000000003, "time": 43.38757050037384}, "59e0117b7d": {"quality": 0.21752136752136753, "cost": 0.006857488, "time": 139.82263526916503}, "59f515d0da": {"quality": 0.2064102564102564, "cost": 0.00388749, "time": 108.1351862192154}, "59f887b67c": {"quality": 0.0, "cost": 0.010208398, "time": 146.34244396686552}, "5a22920db4": {"quality": 0.1987179487179487, "cost": 0.0018006569999999998, "time": 75.78275735378266}, "5a35020d45": {"quality": 0.1987179487179487, "cost": 0.0060887910000000005, "time": 109.82949197292328}, "5aa71bb88a": {"quality": 0.12094017094017094, "cost": 0.000638436, "time": 67.15015056133271}, "5b10fbdbe1": {"quality": 0.21752136752136753, "cost": 0.013089312000000002, "time": 128.6064488887787}, "5b4ad39a9e": {"quality": 0.11752136752136753, "cost": 0.0032681159999999997, "time": 99.70189120769501}, "5bade9eb85": {"quality": 0.18482905982905984, "cost": 0.008712719999999998, "time": 120.3732186794281}, "5be16744bf": {"quality": 0.1952991452991453, "cost": 0.013749790000000001, "time": 91.71922521591186}, "5c0db11303": {"quality": 0.058333333333333334, "cost": 0.002397702, "time": 93.91862483024596}, "5c53feccd9": {"quality": 0.023076923076923078, "cost": 0.015926556, "time": 118.78805196285248}, "5c77c7c2b2": {"quality": 0.15, "cost": 0.008434941000000001, "time": 85.08626408576964}, "5d072194b8": {"quality": 0.11752136752136753, "cost": 0.007648864000000002, "time": 104.28334321975709}, "5d79b50feb": {"quality": 0.14038461538461539, "cost": 0.013589224, "time": 84.18065688610076}, "5dc216cd6b": {"quality": 0.10982905982905983, "cost": 0.010508788, "time": 83.96197824478149}, "5dd68c1b8f": {"quality": 0.16944444444444445, "cost": 0.001444968, "time": 54.59350550174713}, "5de4a882c1": {"quality": 0.21752136752136753, "cost": 0.008734155, "time": 75.31518497467042}, "5deeeb223f": {"quality": 0.24316239316239316, "cost": 0.005724273, "time": 115.48380098342895}, "5e2f03b962": {"quality": 0.1876068376068376, "cost": 0.0007302779999999999, "time": 48.90210340023041}, "5ea2fab380": {"quality": 0.05149572649572649, "cost": 0.0048052319999999996, "time": 47.63824257850647}, "5eb3bb525b": {"quality": 0.0702991452991453, "cost": 0.008967890000000001, "time": 96.21212074756622}, "5ec3832817": {"quality": 0.0, "cost": 0.008163805, "time": 148.1108601331711}, "5f0199e07b": {"quality": 0.025, "cost": 0.00068568, "time": 49.921340346336365}, "5f37b3902b": {"quality": 0.04038461538461539, "cost": 0.007223498, "time": 77.24008927345275}, "60cb623c53": {"quality": 0.04871794871794872, "cost": 0.002396232, "time": 54.30748798847199}, "612e546d71": {"quality": 0.10705128205128206, "cost": 0.008148234, "time": 151.1273174762726}, "6160bfb439": {"quality": 0.11474358974358972, "cost": 0.010103339999999999, "time": 148.44622106552123}, "6178f33808": {"quality": 0.04722222222222222, "cost": 0.006706226999999999, "time": 153.97144429683686}, "619b48dde9": {"quality": 0.21752136752136753, "cost": 0.006427425999999999, "time": 120.5339899301529}, "6268ac658c": {"quality": 0.24316239316239316, "cost": 0.01069585, "time": 144.7754723072052}, "628f34aace": {"quality": 0.2098290598290598, "cost": 0.005102127, "time": 151.19322457313538}, "630d1ecda0": {"quality": 0.0, "cost": 0.00503412, "time": 158.05562288761138}, "63a0aaebed": {"quality": 0.15833333333333333, "cost": 0.001865547, "time": 88.02101895809173}, "63f392465f": {"quality": 0.18333333333333332, "cost": 0.0020362590000000003, "time": 114.81161072254181}, "6527f214c3": {"quality": 0.125, "cost": 0.0020534459999999996, "time": 107.1269181728363}, "652c0f4bdf": {"quality": 0.1, "cost": 0.013503214, "time": 140.5670464038849}, "6533c85913": {"quality": 0.10641025641025642, "cost": 0.009929774999999998, "time": 122.77552180290222}, "65627426e0": {"quality": 0.15, "cost": 0.000700312, "time": 55.69904806613922}, "65801893b4": {"quality": 0.025, "cost": 0.01106166, "time": 113.8573011636734}, "65b76da9c6": {"quality": 0.12222222222222223, "cost": 0.007336156, "time": 88.77893948554993}, "65be1c1306": {"quality": 0.06752136752136753, "cost": 0.001988206, "time": 34.18720245361328}, "65e0216208": {"quality": 0.05, "cost": 0.008013333, "time": 111.02614188194275}, "65eee615d7": {"quality": 0.1987179487179487, "cost": 0.000519715, "time": 32.74071106910706}, "66277da52f": {"quality": 0.09871794871794873, "cost": 0.0036039389999999996, "time": 110.55973320007324}, "66750c0934": {"quality": 0.21752136752136753, "cost": 0.007508133, "time": 85.75274183750153}, "66776ec181": {"quality": 0.10641025641025642, "cost": 0.006989541, "time": 111.77394149303436}, "67632141f6": {"quality": 0.04038461538461539, "cost": 0.004542882, "time": 63.92582683563233}, "677deb302a": {"quality": 0.1814102564102564, "cost": 0.007409720999999999, "time": 111.17820315361024}, "67868fcff6": {"quality": 0.21752136752136753, "cost": 0.011676892000000001, "time": 81.84591567516327}, "67aad9ea16": {"quality": 0.2098290598290598, "cost": 0.007452915, "time": 126.7279718399048}, "67bab6732d": {"quality": 0.10555555555555557, "cost": 0.01121626, "time": 84.01423738002777}, "67fe399cf1": {"quality": 0.21752136752136753, "cost": 0.007087558, "time": 80.51065149307252}, "6846bd8fb3": {"quality": 0.08141025641025641, "cost": 0.006947139, "time": 85.74786493778228}, "68583552fb": {"quality": 0.0, "cost": 0.0053533439999999995, "time": 93.39369978904725}, "689e327daf": {"quality": 0.08333333333333334, "cost": 0.000799428, "time": 57.74340398311615}, "69b3b67de6": {"quality": 0.23974358974358972, "cost": 0.008200215, "time": 95.98227195739746}, "69bf3f6ba0": {"quality": 0.21752136752136753, "cost": 0.008827848, "time": 102.32003858089448}, "69f90e610f": {"quality": 0.22585470085470086, "cost": 0.011046368, "time": 121.27799794673919}, "6a022c3f73": {"quality": 0.21944444444444444, "cost": 0.004195854, "time": 99.68456645011902}, "6a10c53ad8": {"quality": 0.32863247863247863, "cost": 0.012164633000000001, "time": 119.7128321170807}, "6a6348f69d": {"quality": 0.025, "cost": 0.0010719339999999999, "time": 79.13316841125489}, "6a8726145c": {"quality": 0.1876068376068376, "cost": 0.0014679629999999996, "time": 77.99199786186219}, "6a8a675442": {"quality": 0.22094017094017093, "cost": 0.005341439999999999, "time": 90.32394058704377}, "6aac59742a": {"quality": 0.0, "cost": 0.009132285, "time": 137.44288988113402}, "6ac193c88f": {"quality": 0.25085470085470085, "cost": 0.006668448, "time": 121.86276133060456}, "6ae9e9de0b": {"quality": 0.21752136752136753, "cost": 0.0077003290000000005, "time": 125.54181215763091}, "6b0c585f5c": {"quality": 0.21752136752136753, "cost": 0.006663018, "time": 122.58973615169526}, "6b3c16def2": {"quality": 0.18482905982905984, "cost": 0.001533927, "time": 51.53218412399292}, "6c05c47050": {"quality": 0.1987179487179487, "cost": 0.003248111999999999, "time": 96.47181532382965}, "6c3667811b": {"quality": 0.21474358974358973, "cost": 0.005244696, "time": 89.6854020357132}, "6cc813aa68": {"quality": 0.14807692307692308, "cost": 0.005938046000000001, "time": 58.51538195610046}, "6cd78cac7e": {"quality": 0.09444444444444444, "cost": 0.007387408, "time": 101.36276533603669}, "6d20c6ace0": {"quality": 0.21752136752136753, "cost": 0.024144216, "time": 121.96203644275664}, "6d67c56ba6": {"quality": 0.17307692307692307, "cost": 0.009393948, "time": 150.0689915418625}, "6db70dc3b6": {"quality": 0.04038461538461539, "cost": 0.0015085300000000001, "time": 94.44009244441986}, "6e0690f576": {"quality": 0.0, "cost": 0.001440486, "time": 74.4156935930252}, "6e3db7ec5e": {"quality": 0.21752136752136753, "cost": 0.009058176, "time": 141.13843698501586}, "6e62bbb47f": {"quality": 0.15982905982905982, "cost": 0.0032870549999999997, "time": 139.53261284828187}, "6e859bfae6": {"quality": 0.05, "cost": 0.004340145, "time": 162.00480234622955}, "6e93514f45": {"quality": 0.0, "cost": 0.0047780819999999995, "time": 95.28718383312224}, "6eae47102b": {"quality": 0.21752136752136753, "cost": 0.008071342, "time": 93.64182848930359}, "6ecf93c479": {"quality": 0.17307692307692307, "cost": 0.0028209449999999996, "time": 103.17148315906525}, "6ef3b7127e": {"quality": 0.21752136752136753, "cost": 0.006768256, "time": 68.10584411621093}, "6f323f80c7": {"quality": 0.0, "cost": 0.00037804, "time": 104.40542347431182}, "6f60a05c33": {"quality": 0.15833333333333333, "cost": 0.0037825470000000003, "time": 92.0291631937027}, "6fbdd8b57c": {"quality": 0.1, "cost": 0.002123712, "time": 95.29761900901795}, "6fd6046c4b": {"quality": 0.18333333333333332, "cost": 0.005121717, "time": 132.3336772441864}, "6fe0b3f929": {"quality": 0.0, "cost": 0.006223385999999999, "time": 68.93117754459381}, "6ff4f667f8": {"quality": 0.19252136752136753, "cost": 0.006652730000000001, "time": 87.94405431747437}, "700474dfbd": {"quality": 0.15833333333333333, "cost": 0.0028382069999999997, "time": 102.84161474704743}, "700ab1d309": {"quality": 0.023076923076923078, "cost": 0.017122988, "time": 117.11154954433441}, "7040e83d52": {"quality": 0.24166666666666664, "cost": 0.019616124999999998, "time": 101.78968648910522}, "704209377f": {"quality": 0.0876068376068376, "cost": 0.005003547, "time": 138.1696399450302}, "70b666e371": {"quality": 0.18482905982905984, "cost": 0.004856766, "time": 104.8691722393036}, "70c850e039": {"quality": 0.18333333333333332, "cost": 0.002363835, "time": 67.29307141304017}, "7112a7e64c": {"quality": 0.25085470085470085, "cost": 0.0058677550000000005, "time": 63.623961353302}, "7114013f0c": {"quality": 0.16538461538461538, "cost": 0.0022517069999999995, "time": 125.79093027114868}, "715070d0ca": {"quality": 0.025, "cost": 0.006732683999999999, "time": 135.16606471538546}, "71b615468b": {"quality": 0.09252136752136753, "cost": 0.010796074, "time": 94.77488939762115}, "71ed893462": {"quality": 0.19444444444444442, "cost": 0.011514972000000002, "time": 137.31028594970704}, "722d41b2f8": {"quality": 0.125, "cost": 0.006919008000000001, "time": 110.66710319519044}, "723fd5589a": {"quality": 0.125, "cost": 0.012675105000000002, "time": 105.89904611110688}, "7250da0f41": {"quality": 0.0, "cost": 0.0072467339999999995, "time": 74.25018970966339}, "7260a96349": {"quality": 0.10641025641025642, "cost": 0.006264144, "time": 160.2630994796753}, "7347cf0308": {"quality": 0.0, "cost": 0.003343845, "time": 72.2089759349823}, "736e652158": {"quality": 0.0, "cost": 0.005920191, "time": 146.5679278612137}, "738364d6a2": {"quality": 0.0, "cost": 0.009448194, "time": 159.01038644313812}, "739b1f81dc": {"quality": 0.1987179487179487, "cost": 0.0017973389999999998, "time": 73.72460424900055}, "73c6240c29": {"quality": 0.1, "cost": 0.005322242999999999, "time": 159.0038095474243}, "73fc2767b9": {"quality": 0.2064102564102564, "cost": 0.0037065989999999997, "time": 126.9765938282013}, "7445d99939": {"quality": 0.1814102564102564, "cost": 0.007021181999999999, "time": 111.21800615787507}, "7466a5f424": {"quality": 0.3175213675213675, "cost": 0.015386124, "time": 150.73878495693208}, "74cc4b1bc4": {"quality": 0.04038461538461539, "cost": 0.006254895, "time": 117.36351308822631}, "74d7f64b8c": {"quality": 0.0, "cost": 0.006549831000000001, "time": 151.8392071723938}, "751869cbec": {"quality": 0.015384615384615385, "cost": 0.007969879, "time": 134.38790624141694}, "7524905580": {"quality": 0.2098290598290598, "cost": 0.004221619, "time": 93.99037828445435}, "752d9649f2": {"quality": 0.25085470085470085, "cost": 0.01328176, "time": 121.08108768463134}, "7558c9722d": {"quality": 0.2064102564102564, "cost": 0.008134605999999999, "time": 120.57110471725464}, "75ca9cd4f8": {"quality": 0.1876068376068376, "cost": 0.0004974599999999999, "time": 68.25030062198638}, "7604c0aa13": {"quality": 0.015384615384615385, "cost": 0.010546463999999998, "time": 90.40108380317687}, "765dbc6ad5": {"quality": 0.21752136752136753, "cost": 0.003435049, "time": 94.19224355220794}, "7707e6e7e3": {"quality": 0.11474358974358972, "cost": 0.010274625999999999, "time": 94.13086493015288}, "7765576286": {"quality": 0.2098290598290598, "cost": 0.008545683, "time": 127.77431318759919}, "77983b6105": {"quality": 0.0, "cost": 0.002284218, "time": 68.26882412433625}, "77b5740025": {"quality": 0.015384615384615385, "cost": 0.006498776, "time": 106.45389134883881}, "77c02b00c1": {"quality": 0.257051282051282, "cost": 0.010423473999999999, "time": 126.61633729934692}, "77c6a9703a": {"quality": 0.2098290598290598, "cost": 0.007362192, "time": 108.2735918521881}, "77f293b737": {"quality": 0.21752136752136753, "cost": 0.009682152999999999, "time": 136.6608712911606}, "7801da66b9": {"quality": 0.0, "cost": 0.004307142, "time": 92.96408789157867}, "7862ea67cb": {"quality": 0.15149572649572648, "cost": 0.0014619539999999997, "time": 74.10243089199066}, "786e5d0af5": {"quality": 0.22094017094017093, "cost": 0.009905354999999998, "time": 129.44383957386017}, "795d119bc7": {"quality": 0.0, "cost": 0.006895664000000001, "time": 81.85152049064637}, "7989343d94": {"quality": 0.11666666666666667, "cost": 0.0030767639999999996, "time": 104.29451589584352}, "79fad58f07": {"quality": 0.21752136752136753, "cost": 0.002502802, "time": 51.48524146080017}, "7a207b42a8": {"quality": 0.023076923076923078, "cost": 0.008630430000000001, "time": 131.3380735397339}, "7a58d3472b": {"quality": 0.06752136752136753, "cost": 0.012301829, "time": 96.64494693279266}, "7a7cc658c8": {"quality": 0.06944444444444445, "cost": 0.009218845, "time": 130.32710280418394}, "7b024a2966": {"quality": 0.14807692307692308, "cost": 0.009636592999999999, "time": 127.10254328250885}, "7b6dc3702e": {"quality": 0.11752136752136753, "cost": 0.01032963, "time": 98.65728087425232}, "7b6f44618e": {"quality": 0.21752136752136753, "cost": 0.012669284, "time": 119.66786665916442}, "7b74b23910": {"quality": 0.06538461538461539, "cost": 0.012630904, "time": 93.18642947673797}, "7b9cc96081": {"quality": 0.14807692307692308, "cost": 0.000405328, "time": 79.7800312757492}, "7c45c61d8d": {"quality": 0.21752136752136753, "cost": 0.006698508, "time": 122.86696994304657}, "7c89a2b69e": {"quality": 0.025, "cost": 0.012249355, "time": 127.1623036623001}, "7d44f0959d": {"quality": 0.21752136752136753, "cost": 0.006413139, "time": 93.95755279064178}, "7d60c38c5c": {"quality": 0.2098290598290598, "cost": 0.0017982599999999999, "time": 60.89626235961914}, "7d67e14414": {"quality": 0.11752136752136753, "cost": 0.0037651139999999995, "time": 123.35564484596253}, "7daf7ff182": {"quality": 0.13974358974358975, "cost": 0.011279536, "time": 93.00705258846284}, "7e0ad1c9c1": {"quality": 0.07371794871794872, "cost": 0.00508167, "time": 85.57242858409882}, "7ed07ad40a": {"quality": 0.2098290598290598, "cost": 0.00644937, "time": 84.29863061904908}, "7fa67a7656": {"quality": 0.07307692307692308, "cost": 0.007675500000000001, "time": 69.2791738986969}, "806881adcb": {"quality": 0.2064102564102564, "cost": 0.008654110999999999, "time": 86.79695003032684}, "80a1d9c2f3": {"quality": 0.15833333333333333, "cost": 0.005246394, "time": 105.42429778575897}, "80be7df955": {"quality": 0.0, "cost": 0.0014326019999999998, "time": 58.2230907201767}, "80bf60c422": {"quality": 0.16538461538461538, "cost": 0.001969395, "time": 61.00985796451569}, "81333c7a33": {"quality": 0.1987179487179487, "cost": 0.018448848000000004, "time": 86.57931089401245}, "813e75210b": {"quality": 0.23632478632478632, "cost": 0.002293942, "time": 33.89890332221985}, "815e7116df": {"quality": 0.041025641025641026, "cost": 0.00256722, "time": 102.2627355337143}, "816068ff07": {"quality": 0.15833333333333333, "cost": 0.0041238989999999994, "time": 127.76587257385255}, "81a4f42fd9": {"quality": 0.21752136752136753, "cost": 0.005694090000000001, "time": 59.030240178108215}, "828ccea2d3": {"quality": 0.04038461538461539, "cost": 0.008288664, "time": 109.34065868854523}, "829df73946": {"quality": 0.2098290598290598, "cost": 0.002660831, "time": 58.90988037586212}, "831728b179": {"quality": 0.0, "cost": 0.005039063999999999, "time": 71.040651845932}, "831e8b8be5": {"quality": 0.1987179487179487, "cost": 0.002362275, "time": 99.75399866104127}, "8357183895": {"quality": 0.0, "cost": 0.008517975, "time": 130.41329088211057}, "8392a6083a": {"quality": 0.2098290598290598, "cost": 0.013261482, "time": 129.61621696949004}, "83b244c163": {"quality": 0.2098290598290598, "cost": 0.0017275300000000001, "time": 63.67854707241058}, "83c9e66ec6": {"quality": 0.2237179487179487, "cost": 0.007498675, "time": 99.27389492988587}, "842c0d1062": {"quality": 0.20705128205128204, "cost": 0.005142987, "time": 166.0325870513916}, "846bed2aa7": {"quality": 0.24252136752136752, "cost": 0.006434445, "time": 98.26402361392975}, "847fd49235": {"quality": 0.17863247863247866, "cost": 0.004663961999999999, "time": 112.9008763551712}, "84dc98be95": {"quality": 0.19444444444444442, "cost": 0.007237588999999999, "time": 108.35145225524903}, "8519bef585": {"quality": 0.015384615384615385, "cost": 0.0024311339999999997, "time": 63.10895071029663}, "8572c6af3a": {"quality": 0.22863247863247865, "cost": 0.005362141000000001, "time": 159.52487354278566}, "85c94a5505": {"quality": 0.05, "cost": 0.0025202259999999995, "time": 34.404055738449095}, "85e8eaed6e": {"quality": 0.06538461538461539, "cost": 0.004890077999999999, "time": 158.11657013893125}, "862183bfb9": {"quality": 0.21752136752136753, "cost": 0.007488814999999999, "time": 120.0406931400299}, "8668f65f05": {"quality": 0.21752136752136753, "cost": 0.009651689000000001, "time": 144.70214662551882}, "870e2f87b4": {"quality": 0.21752136752136753, "cost": 0.013074957000000002, "time": 150.40512261390685}, "87c1b31c82": {"quality": 0.15982905982905982, "cost": 0.011328134, "time": 152.22725927829742}, "88436e05a9": {"quality": 0.09252136752136753, "cost": 0.010687336, "time": 153.95657515525818}, "887ad124e1": {"quality": 0.1987179487179487, "cost": 0.0030405569999999997, "time": 113.64234898090362}, "8886cb3082": {"quality": 0.2098290598290598, "cost": 0.009527662999999999, "time": 144.03201706409453}, "8940398bf1": {"quality": 0.04038461538461539, "cost": 0.001635585, "time": 86.64000837802887}, "8941621423": {"quality": 0.1737179487179487, "cost": 0.006337113, "time": 113.57799446582794}, "8961e4d901": {"quality": 0.0, "cost": 0.007177176, "time": 104.54208600521088}, "8974aa89a0": {"quality": 0.2098290598290598, "cost": 0.001461674, "time": 45.66577224731445}, "89a289907e": {"quality": 0.25085470085470085, "cost": 0.008718126999999999, "time": 113.69486925601959}, "89a35a09b1": {"quality": 0.21752136752136753, "cost": 0.007729346999999999, "time": 109.72029886245727}, "89bc21961a": {"quality": 0.18482905982905984, "cost": 0.000487962, "time": 54.24717800617218}, "89fbefd150": {"quality": 0.23974358974358972, "cost": 0.00825181, "time": 111.73369183540345}, "8a37c82283": {"quality": 0.23974358974358972, "cost": 0.007666522, "time": 83.14605205059051}, "8aaadb8649": {"quality": 0.025, "cost": 0.006027930000000001, "time": 66.97984294891357}, "8acd758b7f": {"quality": 0.2098290598290598, "cost": 0.004582077, "time": 86.44987049102784}, "8b721bbc6f": {"quality": 0.21752136752136753, "cost": 0.007828845000000001, "time": 117.83376302719117}, "8b90f4b639": {"quality": 0.0, "cost": 0.00041072999999999994, "time": 49.55189027786255}, "8bbbe0f52a": {"quality": 0.1814102564102564, "cost": 0.006133866, "time": 83.5189700126648}, "8bc184f385": {"quality": 0.1702991452991453, "cost": 0.007205481, "time": 86.5741204738617}, "8bf5c3eadc": {"quality": 0.025, "cost": 0.006170141999999999, "time": 83.84268100261687}, "8c195addc7": {"quality": 0.16944444444444445, "cost": 0.01389775, "time": 90.39201626777648}, "8c9881972c": {"quality": 0.0626068376068376, "cost": 0.0028528439999999998, "time": 86.04524366855621}, "8cf8b81d84": {"quality": 0.125, "cost": 0.0008481899999999999, "time": 79.2455335855484}, "8d79e03266": {"quality": 0.21752136752136753, "cost": 0.008664971, "time": 108.97852082252503}, "8d90814b94": {"quality": 0.21752136752136753, "cost": 0.009054151, "time": 130.47983191013338}, "8e33fac90f": {"quality": 0.30982905982905984, "cost": 0.010998730000000002, "time": 130.90265057086944}, "8e5842ccbd": {"quality": 0.14038461538461539, "cost": 0.003239019, "time": 70.15636944770813}, "8f4caddfe6": {"quality": 0.21752136752136753, "cost": 0.014952506, "time": 105.19670691490174}, "8f4edde3f0": {"quality": 0.3175213675213675, "cost": 0.017418426, "time": 126.13828411102295}, "900a58f984": {"quality": 0.2064102564102564, "cost": 0.006433365999999999, "time": 98.91697227954865}, "9025e2480f": {"quality": 0.1987179487179487, "cost": 0.006619497, "time": 78.82729182243347}, "9028588af4": {"quality": 0.3175213675213675, "cost": 0.00501719, "time": 37.611924695968625}, "9059fd80ad": {"quality": 0.04038461538461539, "cost": 0.0018876219999999998, "time": 75.3997132062912}, "90d5e40c1b": {"quality": 0.0, "cost": 0.006230459999999999, "time": 77.66650586128236}, "90d9a86a2a": {"quality": 0.0, "cost": 0.001509345, "time": 48.08298280239106}, "90ed9312e1": {"quality": 0.16944444444444445, "cost": 0.008080153999999999, "time": 122.39040281772614}, "90ff13783c": {"quality": 0.24316239316239316, "cost": 0.010701109, "time": 137.6493337869644}, "90ff8eb055": {"quality": 0.0702991452991453, "cost": 0.009148667999999999, "time": 137.7864047050476}, "9104e31369": {"quality": 0.06944444444444445, "cost": 0.012645782999999999, "time": 140.74590210914613}, "918983323f": {"quality": 0.12307692307692308, "cost": 0.0011798999999999998, "time": 42.641357612609866}, "91beb0cac1": {"quality": 0.125, "cost": 0.000758454, "time": 83.41916146278382}, "91c800af6b": {"quality": 0.04038461538461539, "cost": 0.011276811, "time": 143.70840940475466}, "91dd8884db": {"quality": 0.20705128205128204, "cost": 0.005775448000000001, "time": 141.27923700809478}, "924f128b3c": {"quality": 0.21752136752136753, "cost": 0.013985312, "time": 127.0183181285858}, "9288642e53": {"quality": 0.1, "cost": 0.006031014000000001, "time": 71.42156167030335}, "92c4137fb1": {"quality": 0.25085470085470085, "cost": 0.006908012999999999, "time": 145.73949379920958}, "92c9dcd43b": {"quality": 0.21752136752136753, "cost": 0.008693151, "time": 141.44435067176818}, "92f45e5cc7": {"quality": 0.23974358974358972, "cost": 0.0006816839999999999, "time": 69.65455045700074}, "93011c0821": {"quality": 0.16752136752136754, "cost": 0.012118556999999999, "time": 129.1434654712677}, "9303149ba4": {"quality": 0.04807692307692308, "cost": 0.007609685, "time": 148.0766399145126}, "933b4d17dd": {"quality": 0.22863247863247865, "cost": 0.004478119000000001, "time": 98.24304263591766}, "94010928c6": {"quality": 0.1814102564102564, "cost": 0.006916301999999999, "time": 109.81094932556152}, "9403809e44": {"quality": 0.21752136752136753, "cost": 0.011799824, "time": 132.59149780273435}, "943baaea0c": {"quality": 0.17307692307692307, "cost": 0.011566795000000001, "time": 117.94117500782014}, "94569f177a": {"quality": 0.015384615384615385, "cost": 0.002410395, "time": 125.42731764316558}, "9466542023": {"quality": 0.03333333333333333, "cost": 0.0057622739999999995, "time": 136.34905714988707}, "947e28ef2e": {"quality": 0.06538461538461539, "cost": 0.0031674240000000003, "time": 124.9436069726944}, "94ac356663": {"quality": 0.1737179487179487, "cost": 0.007041248, "time": 104.0579169511795}, "94dff9a424": {"quality": 0.1987179487179487, "cost": 0.005677325, "time": 105.50971393585205}, "9508356a2e": {"quality": 0.14807692307692308, "cost": 0.006912872999999999, "time": 129.4876657009125}, "956bdcc254": {"quality": 0.2064102564102564, "cost": 0.015445604, "time": 125.50174486637115}, "9594b0c783": {"quality": 0.2098290598290598, "cost": 0.003143199, "time": 109.28865358829498}, "964c671f18": {"quality": 0.2098290598290598, "cost": 0.011780327, "time": 147.56322202682497}, "9679fe2b69": {"quality": 0.4098290598290598, "cost": 0.003103125, "time": 112.40209789276122}, "968fc95038": {"quality": 0.21752136752136753, "cost": 0.00356376, "time": 109.65150537490845}, "96b487c724": {"quality": 0.07371794871794872, "cost": 0.0006113920000000001, "time": 69.84607322216033}, "96e85f9af4": {"quality": 0.058333333333333334, "cost": 0.0037689600000000005, "time": 115.5232797384262}, "96f87d6483": {"quality": 0.19444444444444442, "cost": 0.012450801, "time": 146.1588816165924}, "972c83b002": {"quality": 0.14807692307692308, "cost": 0.007372287, "time": 112.55022113323211}, "975bc44958": {"quality": 0.24871794871794872, "cost": 0.0024022649999999998, "time": 111.18246881961822}, "977a4d6b6b": {"quality": 0.33974358974358976, "cost": 0.013285368000000002, "time": 138.42745580673215}, "97ad4cd41a": {"quality": 0.025, "cost": 0.008055923000000001, "time": 147.88769648075103}, "97bc30bd83": {"quality": 0.19252136752136753, "cost": 0.010554706, "time": 107.51760149002075}, "97e1d0db92": {"quality": 0.06538461538461539, "cost": 0.004098666000000001, "time": 98.48549189567566}, "981da9ba40": {"quality": 0.09871794871794873, "cost": 0.008754687, "time": 150.4159719467163}, "98c1ea89f3": {"quality": 0.1987179487179487, "cost": 0.004756095, "time": 144.49644501209258}, "98eca2c65c": {"quality": 0.2098290598290598, "cost": 0.007576088, "time": 113.79488031864167}, "98ecf1a157": {"quality": 0.2098290598290598, "cost": 0.0006460169999999999, "time": 66.48974347114563}, "9927dc270b": {"quality": 0.22863247863247865, "cost": 0.008019299, "time": 142.80323901176453}, "99546d91e4": {"quality": 0.2098290598290598, "cost": 0.0023279909999999997, "time": 109.73371806144715}, "99cb0ba736": {"quality": 0.0, "cost": 0.001753854, "time": 80.39895787239075}, "99e44ab9b2": {"quality": 0.2064102564102564, "cost": 0.007417979999999999, "time": 142.21234567165374}, "9a0145c9b5": {"quality": 0.07371794871794872, "cost": 0.003074034, "time": 115.01807222366332}, "9a57ea3f89": {"quality": 0.1952991452991453, "cost": 0.007409801999999998, "time": 133.90688972473146}, "9a8420a0b3": {"quality": 0.0, "cost": 0.0068175779999999995, "time": 152.5392238378525}, "9aa32e6c96": {"quality": 0.21752136752136753, "cost": 0.01103525, "time": 141.08924725055692}, "9aa4abfb50": {"quality": 0.0, "cost": 0.0034209360000000003, "time": 89.89876456260681}, "9ada932bf5": {"quality": 0.21752136752136753, "cost": 0.008047750999999999, "time": 144.1339359998703}, "9b6d4915f3": {"quality": 0.0, "cost": 0.00463566, "time": 54.38807971477509}, "9bae5bafc1": {"quality": 0.18333333333333332, "cost": 0.013702148, "time": 98.45531895160676}, "9c549db0a7": {"quality": 0.0, "cost": 0.010721709000000001, "time": 79.0128826379776}, "9c595a2bc9": {"quality": 0.025, "cost": 0.007724396, "time": 148.95668649673462}, "9c85f8cfcb": {"quality": 0.015384615384615385, "cost": 0.0023850119999999997, "time": 67.28472025394439}, "9c8cc46e6c": {"quality": 0.06538461538461539, "cost": 0.002001585, "time": 77.71750602722167}, "9c97d35a30": {"quality": 0.2098290598290598, "cost": 0.005077656, "time": 118.16123571395875}, "9ca354a53e": {"quality": 0.0, "cost": 0.003021384, "time": 101.3728716135025}, "9ce2c3fd98": {"quality": 0.0, "cost": 0.0047293560000000005, "time": 129.50395069122314}, "9d18cd0737": {"quality": 0.07307692307692308, "cost": 0.004835114999999999, "time": 82.27081375122071}, "9d7142e7b4": {"quality": 0.16944444444444445, "cost": 0.008467907, "time": 147.60730669498443}, "9d778daa24": {"quality": 0.03333333333333333, "cost": 0.007418613, "time": 153.25245223045349}, "9e06360bc9": {"quality": 0.2098290598290598, "cost": 0.008218711, "time": 113.20028014183045}, "9fb157be35": {"quality": 0.1, "cost": 0.008187586, "time": 116.65422949790954}, "9fc44fdeb1": {"quality": 0.0, "cost": 0.009764671999999999, "time": 163.913742351532}, "9ffaa26d5a": {"quality": 0.21752136752136753, "cost": 0.006876598000000001, "time": 107.02223331928253}, "a041e7777a": {"quality": 0.17649572649572648, "cost": 0.0054005519999999994, "time": 107.31747977733612}, "a0b81be5b4": {"quality": 0.21752136752136753, "cost": 0.002235789, "time": 84.43192894458771}, "a0c85d260e": {"quality": 0.21752136752136753, "cost": 0.013538011000000003, "time": 134.37746741771696}, "a0dc9f50ac": {"quality": 0.15833333333333333, "cost": 0.0058347719999999985, "time": 73.68754653930664}, "a14c507393": {"quality": 0.0, "cost": 0.0016790159999999998, "time": 85.27930269241332}, "a1881eb481": {"quality": 0.23974358974358972, "cost": 0.006974355, "time": 158.60932910442352}, "a1bb32e6a1": {"quality": 0.04038461538461539, "cost": 0.0019489619999999998, "time": 113.17418549060821}, "a2347e8e9e": {"quality": 0.08482905982905983, "cost": 0.003918969, "time": 112.96018514633178}, "a2aa082d14": {"quality": 0.0, "cost": 0.012548850000000002, "time": 97.43302309513092}, "a2cd339ad9": {"quality": 0.09252136752136753, "cost": 0.009654009, "time": 161.78442559242248}, "a2fd03e6a5": {"quality": 0.1876068376068376, "cost": 0.004650046, "time": 46.97034850120544}, "a31e87d7cb": {"quality": 0.2064102564102564, "cost": 0.00146718, "time": 73.75970520973206}, "a344b2d79a": {"quality": 0.2098290598290598, "cost": 0.008522825, "time": 167.94721865653992}, "a3c0ea3342": {"quality": 0.3175213675213675, "cost": 0.007620170000000001, "time": 166.26401495933533}, "a456d75fef": {"quality": 0.0, "cost": 0.003897582, "time": 149.68703374862673}, "a457f6c300": {"quality": 0.0, "cost": 0.007946964, "time": 158.9001291036606}, "a4767e7679": {"quality": 0.0, "cost": 0.002210106, "time": 111.21880123615264}, "a47de025c8": {"quality": 0.0, "cost": 0.003860679, "time": 106.15438146591185}, "a4ad96343d": {"quality": 0.23696581196581196, "cost": 0.009205054, "time": 161.23487601280215}, "a515a9c8cc": {"quality": 0.16944444444444445, "cost": 0.0021845939999999998, "time": 90.79483761787415}, "a5949b76ec": {"quality": 0.0, "cost": 0.0022742099999999996, "time": 77.40827825069428}, "a5ae4dfe66": {"quality": 0.1, "cost": 0.006415026, "time": 117.52768676280975}, "a60dd076b8": {"quality": 0.1564102564102564, "cost": 0.0024921179999999998, "time": 99.6796523809433}, "a6297a6c56": {"quality": 0.125, "cost": 0.0019724219999999997, "time": 83.12536749839782}, "a6460dbb7c": {"quality": 0.2098290598290598, "cost": 0.002657791, "time": 71.54174087047576}, "a6796ed686": {"quality": 0.09444444444444444, "cost": 0.0032055929999999996, "time": 130.10447964668273}, "a6d2b05ec8": {"quality": 0.2098290598290598, "cost": 0.008059675, "time": 133.3959671497345}, "a717c4c535": {"quality": 0.11752136752136753, "cost": 0.012759294000000001, "time": 123.11250400543213}, "a721cd9ebf": {"quality": 0.21752136752136753, "cost": 0.006374966000000001, "time": 93.21334941387177}, "a8090787b1": {"quality": 0.2098290598290598, "cost": 0.004406745, "time": 124.60862724781036}, "a854343d46": {"quality": 0.225, "cost": 0.0063598100000000005, "time": 87.96014966964722}, "a86b137d7f": {"quality": 0.15, "cost": 0.0021869159999999997, "time": 58.69887585639954}, "a88eb1493c": {"quality": 0.0952991452991453, "cost": 0.004839059999999999, "time": 60.44982805252076}, "a88fb984e3": {"quality": 0.21752136752136753, "cost": 0.008328870999999998, "time": 131.90409784317018}, "a94e2e5f57": {"quality": 0.04807692307692308, "cost": 0.006114, "time": 102.40570263862611}, "a95b4a6dd0": {"quality": 0.0452991452991453, "cost": 0.008242788000000001, "time": 101.27096877098083}, "a9621ea4e6": {"quality": 0.22863247863247865, "cost": 0.00707055, "time": 100.2644911289215}, "a96e22379d": {"quality": 0.13482905982905985, "cost": 0.004674597, "time": 135.9614454984665}, "a9721a0a50": {"quality": 0.18482905982905984, "cost": 0.006856040000000001, "time": 74.35422718524933}, "aa08180e36": {"quality": 0.21752136752136753, "cost": 0.009903308, "time": 141.78917632102966}, "aa38702a02": {"quality": 0.058333333333333334, "cost": 0.004223736, "time": 108.32900211811065}, "aa8187c023": {"quality": 0.0, "cost": 0.006725136, "time": 115.54149260520936}, "aadbfc418b": {"quality": 0.04807692307692308, "cost": 0.003954492, "time": 115.65469679832458}, "ab1c706436": {"quality": 0.09871794871794873, "cost": 0.007816556999999998, "time": 157.3217898607254}, "ab43b02cb0": {"quality": 0.0, "cost": 0.010739148, "time": 88.10484538078308}, "aba21780bc": {"quality": 0.09594017094017093, "cost": 0.001212108, "time": 68.86885101795197}, "abbca95f00": {"quality": 0.1841880341880342, "cost": 0.00179952, "time": 77.29247143268586}, "ac208e7a1d": {"quality": 0.21752136752136753, "cost": 0.009940174000000001, "time": 106.78546745777129}, "ac7fcf90e2": {"quality": 0.059829059829059825, "cost": 0.008491644, "time": 163.5559736728668}, "ac828ffe70": {"quality": 0.025, "cost": 0.000761398, "time": 72.60451011657715}, "ac9fdc1550": {"quality": 0.2098290598290598, "cost": 0.012285289000000001, "time": 112.57146043777465}, "ad328d5108": {"quality": 0.1611111111111111, "cost": 0.001292802, "time": 134.6257879257202}, "ad3efe44c3": {"quality": 0.0, "cost": 0.002330118, "time": 83.25956127643585}, "ad48432c22": {"quality": 0.0, "cost": 0.009617565, "time": 165.95421760082246}, "ad5187a390": {"quality": 0.22863247863247865, "cost": 0.010695554, "time": 160.50709626674652}, "ad6ebbba8d": {"quality": 0.16752136752136754, "cost": 0.0075785209999999995, "time": 114.89612691402435}, "ad90055ef6": {"quality": 0.0, "cost": 0.003493998, "time": 103.62199125289916}, "ad97c5cee6": {"quality": 0.15149572649572648, "cost": 0.0022562339999999998, "time": 137.5054316520691}, "adab1e0fb1": {"quality": 0.3175213675213675, "cost": 0.00585581, "time": 77.00263237953186}, "ae655ec593": {"quality": 0.11666666666666667, "cost": 0.003635154, "time": 112.42764747142792}, "ae94b172be": {"quality": 0.1987179487179487, "cost": 0.007214128000000001, "time": 117.53457653522491}, "af360c323c": {"quality": 0.0, "cost": 0.0032059769999999996, "time": 116.28576538562774}, "af90567194": {"quality": 0.21752136752136753, "cost": 0.013109858999999998, "time": 144.84608309268953}, "b0156bb6d2": {"quality": 0.25085470085470085, "cost": 0.011064174, "time": 144.33307461738588}, "b03c31ca45": {"quality": 0.09871794871794873, "cost": 0.007032179999999999, "time": 159.58066523075104}, "b0530b98c3": {"quality": 0.08333333333333334, "cost": 0.007914135, "time": 151.2534171819687}, "b0948c05b6": {"quality": 0.16538461538461538, "cost": 0.011687544000000001, "time": 77.90738432407379}, "b18168b9c1": {"quality": 0.1952991452991453, "cost": 0.007746042, "time": 102.36798930168152}, "b1cf8d33e5": {"quality": 0.0702991452991453, "cost": 0.007221741, "time": 96.63562579154969}, "b1dcd7aa24": {"quality": 0.2098290598290598, "cost": 0.009499193, "time": 142.64096357822416}, "b214718d07": {"quality": 0.16538461538461538, "cost": 0.002830191, "time": 105.70124731063842}, "b28925e4b8": {"quality": 0.06538461538461539, "cost": 0.011244171, "time": 117.42571413516998}, "b2b057ba41": {"quality": 0.21752136752136753, "cost": 0.0036646409999999997, "time": 104.08598182201385}, "b2e063499d": {"quality": 0.21752136752136753, "cost": 0.008730451, "time": 99.22217960357665}, "b3369775dc": {"quality": 0.2064102564102564, "cost": 0.012187132, "time": 128.45301840305328}, "b35bf038c2": {"quality": 0.14038461538461539, "cost": 0.0027645659999999996, "time": 118.8479066848755}, "b363b25367": {"quality": 0.10982905982905983, "cost": 0.00807849, "time": 150.05790014266967}, "b3decd5c2f": {"quality": 0.14722222222222223, "cost": 0.0054683579999999996, "time": 114.51114053726197}, "b3f20b706d": {"quality": 0.025, "cost": 0.002810052, "time": 90.74897117614745}, "b4002173ee": {"quality": 0.10641025641025642, "cost": 0.00532104, "time": 71.4259075164795}, "b45fc30d81": {"quality": 0.015384615384615385, "cost": 0.007027874999999999, "time": 148.3045286655426}, "b4a259f6dd": {"quality": 0.2098290598290598, "cost": 0.00846508, "time": 139.42276346683502}, "b52cdb3c6d": {"quality": 0.25085470085470085, "cost": 0.001953663, "time": 101.56977760791779}, "b56c312eda": {"quality": 0.21752136752136753, "cost": 0.008759253, "time": 135.296657371521}, "b5a02bb8ab": {"quality": 0.09252136752136753, "cost": 0.005621411999999999, "time": 126.08018836975097}, "b5e2b41c1c": {"quality": 0.1987179487179487, "cost": 0.004181511, "time": 97.78416235446929}, "b64ddb14f9": {"quality": 0.21752136752136753, "cost": 0.002377122, "time": 40.39987435340882}, "b66118d5f2": {"quality": 0.04038461538461539, "cost": 0.007001360999999999, "time": 141.72445845603943}, "b67107a43e": {"quality": 0.0, "cost": 0.005282744999999999, "time": 134.97200748920443}, "b682a23b89": {"quality": 0.2098290598290598, "cost": 0.004507057, "time": 94.8906276702881}, "b690a1ddd6": {"quality": 0.0, "cost": 0.004254936000000001, "time": 132.1497477531433}, "b796b7ffd3": {"quality": 0.21752136752136753, "cost": 0.008754787, "time": 134.97328844070432}, "b7d0e8557f": {"quality": 0.20982905982905983, "cost": 0.007772019, "time": 136.01008360385896}, "b7f203a0bf": {"quality": 0.09871794871794873, "cost": 0.0074011649999999995, "time": 142.89838807582856}, "b81d5b2bd9": {"quality": 0.04807692307692308, "cost": 0.008468160999999998, "time": 139.87127735614774}, "b8343f05e1": {"quality": 0.04722222222222222, "cost": 0.0071142839999999985, "time": 114.42319309711456}, "b8ab3d2f25": {"quality": 0.21752136752136753, "cost": 0.008466568, "time": 101.01810977458953}, "b8b569172f": {"quality": 0.10705128205128206, "cost": 0.010024281999999999, "time": 115.06630852222443}, "b8b91e375d": {"quality": 0.21752136752136753, "cost": 0.012645420000000001, "time": 146.17070028781893}, "b8c685904d": {"quality": 0.0, "cost": 0.0017556779999999999, "time": 98.82595324516296}, "b8d1903276": {"quality": 0.025, "cost": 0.004277349, "time": 116.25438375473021}, "b91e7fdb29": {"quality": 0.0, "cost": 0.009623747999999998, "time": 172.33287427425387}, "b9770c2261": {"quality": 0.24871794871794872, "cost": 0.0018192599999999998, "time": 68.26051120758056}, "b9bb1e6f8d": {"quality": 0.21752136752136753, "cost": 0.012109797999999998, "time": 153.9570437669754}, "b9da208432": {"quality": 0.22094017094017093, "cost": 0.009226414999999998, "time": 152.71056313514708}, "ba3223f6ac": {"quality": 0.17307692307692307, "cost": 0.003906612, "time": 117.72781562805176}, "bac3d23c31": {"quality": 0.21752136752136753, "cost": 0.011714114000000001, "time": 149.72536618709563}, "bb3ee18de1": {"quality": 0.2064102564102564, "cost": 0.00687103, "time": 112.67253947257996}, "bb6536b0ab": {"quality": 0.12585470085470085, "cost": 0.010593946, "time": 116.58704767227172}, "bb70f60bf1": {"quality": 0.1952991452991453, "cost": 0.0029260320000000003, "time": 113.47788968086243}, "bbba9dd6ae": {"quality": 0.3175213675213675, "cost": 0.005866453000000001, "time": 81.5745332479477}, "bbfba2f2ee": {"quality": 0.0, "cost": 0.0043119479999999995, "time": 144.39048359394076}, "bc3d02f753": {"quality": 0.0, "cost": 0.002622999, "time": 100.0285652399063}, "bc4c1fcc64": {"quality": 0.21752136752136753, "cost": 0.005892321000000001, "time": 80.69622204303741}, "bc60556255": {"quality": 0.025, "cost": 0.000541426, "time": 90.43072290420533}, "bcae7c2fc4": {"quality": 0.0952991452991453, "cost": 0.006106959, "time": 118.90915808677673}, "bcef42e3b0": {"quality": 0.20705128205128204, "cost": 0.003036438, "time": 126.3029891014099}, "bcf4bf7c35": {"quality": 0.09252136752136753, "cost": 0.010682502, "time": 163.63021724224092}, "bcfb273436": {"quality": 0.2675213675213675, "cost": 0.005338022999999999, "time": 154.81484963893888}, "bd99b2fb21": {"quality": 0.21752136752136753, "cost": 0.00784732, "time": 116.22147076129913}, "bdf497196b": {"quality": 0.21752136752136753, "cost": 0.00564954, "time": 81.09978458881378}, "be2ae88f70": {"quality": 0.0, "cost": 0.004892166, "time": 134.319625210762}, "be4740f38f": {"quality": 0.11752136752136753, "cost": 0.011724527999999998, "time": 111.63968887329102}, "bed888d4dc": {"quality": 0.12222222222222223, "cost": 0.007974548999999997, "time": 167.2181126832962}, "bf2a5d2680": {"quality": 0.08141025641025641, "cost": 0.005895564, "time": 116.7006804227829}, "bf7b0a8dc1": {"quality": 0.21752136752136753, "cost": 0.0072987239999999995, "time": 172.42680845260622}, "bfed7670ed": {"quality": 0.08482905982905983, "cost": 0.010263118, "time": 118.4290988445282}, "c06b118e65": {"quality": 0.20705128205128204, "cost": 0.005422905, "time": 111.81110198497773}, "c08a5ad170": {"quality": 0.19252136752136753, "cost": 0.008962954000000002, "time": 153.46751430034638}, "c0d53a20de": {"quality": 0.2098290598290598, "cost": 0.007020855, "time": 151.7084014415741}, "c10e588987": {"quality": 0.1814102564102564, "cost": 0.0070173779999999995, "time": 161.47480256557463}, "c127509a7a": {"quality": 0.30982905982905984, "cost": 0.012152782, "time": 114.87017834186554}, "c13682c7c7": {"quality": 0.21752136752136753, "cost": 0.011820473999999997, "time": 155.98555040359497}, "c13d6e78e9": {"quality": 0.2098290598290598, "cost": 0.015482613, "time": 135.19117062091829}, "c145482664": {"quality": 0.125, "cost": 0.011503632, "time": 157.4289677143097}, "c14ff3144d": {"quality": 0.15833333333333333, "cost": 0.001474128, "time": 73.17416946887971}, "c186182658": {"quality": 0.0702991452991453, "cost": 0.00651766, "time": 114.43175661563873}, "c263c65d0a": {"quality": 0.2098290598290598, "cost": 0.004897752, "time": 161.65783095359802}, "c2949aa902": {"quality": 0.2064102564102564, "cost": 0.006985964999999999, "time": 112.66284742355347}, "c31c9d4d8c": {"quality": 0.058333333333333334, "cost": 0.004036233, "time": 118.73210837841035}, "c31e956b35": {"quality": 0.10705128205128206, "cost": 0.008987883, "time": 156.68281931877135}, "c339463a25": {"quality": 0.11538461538461539, "cost": 0.0039407339999999996, "time": 142.9191876411438}, "c36b525dde": {"quality": 0.2098290598290598, "cost": 0.00295329, "time": 84.8311204433441}, "c3d20f33bf": {"quality": 0.15085470085470087, "cost": 0.009346874, "time": 155.06189365386962}, "c3ec2cec59": {"quality": 0.3175213675213675, "cost": 0.010893408, "time": 121.71417863368988}, "c443a2c1fb": {"quality": 0.09444444444444444, "cost": 0.008210058, "time": 158.09974863529203}, "c4a80d19b3": {"quality": 0.3175213675213675, "cost": 0.010017962999999998, "time": 152.05399761199953}, "c4c2826afd": {"quality": 0.14722222222222223, "cost": 0.008821047, "time": 150.96840312480924}, "c4c94a5527": {"quality": 0.14807692307692308, "cost": 0.007312788, "time": 118.44996173381804}, "c4f3e7665d": {"quality": 0.1, "cost": 0.004909806, "time": 122.77528305053711}, "c58e9652b7": {"quality": 0.03333333333333333, "cost": 0.004275492, "time": 163.37369763851166}, "c5a16b834a": {"quality": 0.24252136752136752, "cost": 0.009288249, "time": 145.43366010189055}, "c5fbe2076f": {"quality": 0.21752136752136753, "cost": 0.016178868, "time": 153.44052112102509}, "c6198f364e": {"quality": 0.21752136752136753, "cost": 0.008840571000000002, "time": 143.01735095977784}, "c691570715": {"quality": 0.025, "cost": 0.004222068, "time": 105.88032670021057}, "c691a29c42": {"quality": 0.06944444444444445, "cost": 0.014146588, "time": 136.24555165767669}, "c6a339987c": {"quality": 0.14807692307692308, "cost": 0.008871827000000002, "time": 124.18036108016967}, "c6a4d256ce": {"quality": 0.11752136752136753, "cost": 0.011863845, "time": 131.42284343242645}, "c76222087e": {"quality": 0.10982905982905983, "cost": 0.007390348000000001, "time": 108.10401859283448}, "c772ff3704": {"quality": 0.10982905982905983, "cost": 0.008256624, "time": 122.45812864303589}, "c7d4ff0c05": {"quality": 0.22863247863247865, "cost": 0.006689247, "time": 135.3987812280655}, "c823f7ab29": {"quality": 0.19252136752136753, "cost": 0.006112934, "time": 110.04468183517456}, "c82b926689": {"quality": 0.21752136752136753, "cost": 0.011001155000000002, "time": 132.11898975372316}, "c82f834e85": {"quality": 0.21752136752136753, "cost": 0.007510131000000001, "time": 88.01416339874268}, "c9320068f9": {"quality": 0.11474358974358972, "cost": 0.008706341999999999, "time": 138.78494324684144}, "c935a33384": {"quality": 0.23974358974358972, "cost": 0.013361947999999998, "time": 128.84306230545045}, "c99f3577c7": {"quality": 0.09444444444444444, "cost": 0.012227506, "time": 131.7928378343582}, "c9d32a0a82": {"quality": 0.1, "cost": 0.00772979, "time": 150.54008300304412}, "ca55c36c3f": {"quality": 0.2098290598290598, "cost": 0.002033097, "time": 109.44059422016144}, "caa7c0bd6b": {"quality": 0.1737179487179487, "cost": 0.007166354999999999, "time": 108.27636570930481}, "cac6b051e9": {"quality": 0.2098290598290598, "cost": 0.007765482000000001, "time": 140.63682539463042}, "cb19d631b2": {"quality": 0.2098290598290598, "cost": 0.0074525500000000005, "time": 109.26569464206696}, "cbb25fb322": {"quality": 0.19444444444444442, "cost": 0.004088256, "time": 146.00942890644075}, "cbb5eb0e74": {"quality": 0.2098290598290598, "cost": 0.007899888, "time": 146.89739501476288}, "cbc32cbeff": {"quality": 0.2098290598290598, "cost": 0.008663157000000001, "time": 144.39441390037535}, "cbd4461293": {"quality": 0.09871794871794873, "cost": 0.0019465299999999997, "time": 53.78504421710968}, "cbe2318045": {"quality": 0.18482905982905984, "cost": 0.00824068, "time": 99.53602702617644}, "cc20ebc768": {"quality": 0.13974358974358975, "cost": 0.008763393999999999, "time": 151.47476081848146}, "cc886fe337": {"quality": 0.05, "cost": 0.006118446, "time": 104.02940430641175}, "ccf72745c1": {"quality": 0.21752136752136753, "cost": 0.00881063, "time": 104.33854112625122}, "cd1d418732": {"quality": 0.0, "cost": 0.005609648, "time": 51.80708644390106}, "cd23c79db1": {"quality": 0.0, "cost": 0.008759461999999999, "time": 130.19892246723174}, "cd64fbfcd9": {"quality": 0.02222222222222222, "cost": 0.006661377, "time": 132.06611769199372}, "cda3e2a4e9": {"quality": 0.19252136752136753, "cost": 0.003251412, "time": 130.11529834270476}, "cdc9ce922f": {"quality": 0.1876068376068376, "cost": 0.00361428, "time": 130.60064585208892}, "cdf0df2f51": {"quality": 0.16538461538461538, "cost": 0.005876955, "time": 170.43609290122987}, "ce281875b4": {"quality": 0.2098290598290598, "cost": 0.002271957, "time": 134.98369066715242}, "ce4bc5f348": {"quality": 0.04038461538461539, "cost": 0.0031947119999999997, "time": 97.7704703092575}, "ce980cf86f": {"quality": 0.125, "cost": 0.003918672, "time": 130.57520353794098}, "cecca90dd2": {"quality": 0.1987179487179487, "cost": 0.004469783999999999, "time": 136.02725315093994}, "cece83de2d": {"quality": 0.10982905982905983, "cost": 0.0060226260000000005, "time": 134.92675962448118}, "cf51e0a888": {"quality": 0.025, "cost": 0.007815034000000002, "time": 141.21634793281555}, "cf9538faf0": {"quality": 0.21752136752136753, "cost": 0.009526646, "time": 136.5719269990921}, "cf9d2e224c": {"quality": 0.11752136752136753, "cost": 0.005909796, "time": 82.14293384552002}, "cfd36f3a8c": {"quality": 0.2064102564102564, "cost": 0.015834602, "time": 133.97862401008604}, "cffa29a6ef": {"quality": 0.1987179487179487, "cost": 0.00053197, "time": 57.04082276821137}, "d03596c3de": {"quality": 0.23974358974358972, "cost": 0.007832440999999999, "time": 161.64306690692902}, "d0f9633442": {"quality": 0.0, "cost": 0.007817082, "time": 133.32561285495757}, "d192872a51": {"quality": 0.06944444444444445, "cost": 0.011960976000000002, "time": 151.55924673080443}, "d1d953cac7": {"quality": 0.023076923076923078, "cost": 0.0069616439999999995, "time": 129.82648131847384}, "d2164c8c4c": {"quality": 0.25085470085470085, "cost": 0.007942218000000001, "time": 106.56984751224518}, "d216eab7d8": {"quality": 0.23974358974358972, "cost": 0.008338906, "time": 110.89031755924225}, "d266c19ac8": {"quality": 0.18482905982905984, "cost": 0.005187306, "time": 137.3413758277893}, "d2af24b59e": {"quality": 0.19252136752136753, "cost": 0.008368941999999999, "time": 107.03122828006744}, "d3a2d50bd7": {"quality": 0.2098290598290598, "cost": 0.004220073, "time": 104.34660010337831}, "d3db4cf84d": {"quality": 0.0, "cost": 0.0043255260000000005, "time": 86.13068716526033}, "d40174fb0b": {"quality": 0.015384615384615385, "cost": 0.004637364, "time": 72.65830745697022}, "d43fafa19e": {"quality": 0.023076923076923078, "cost": 0.004984457999999999, "time": 69.79157807826996}, "d48ead13da": {"quality": 0.04807692307692308, "cost": 0.007356842000000001, "time": 108.29052205085753}, "d5016f4538": {"quality": 0.023076923076923078, "cost": 0.004924038, "time": 63.6337170124054}, "d55e983189": {"quality": 0.2098290598290598, "cost": 0.0023029619999999995, "time": 103.95245385169983}, "d573c2a414": {"quality": 0.04444444444444444, "cost": 0.003798615, "time": 146.4341695547104}, "d58036ba66": {"quality": 0.21752136752136753, "cost": 0.007977542, "time": 101.040083694458}, "d59bacbfe0": {"quality": 0.09252136752136753, "cost": 0.009014214999999999, "time": 152.89927747249604}, "d6040140b9": {"quality": 0.21752136752136753, "cost": 0.013529567000000001, "time": 147.0174176454544}, "d640edd7a7": {"quality": 0.2064102564102564, "cost": 0.0030975000000000004, "time": 118.49127612113952}, "d65185c1a4": {"quality": 0.2098290598290598, "cost": 0.0048733019999999995, "time": 101.63841800689698}, "d690b6d739": {"quality": 0.13760683760683762, "cost": 0.0024723749999999997, "time": 115.86975026130676}, "d6bd3b66ba": {"quality": 0.0, "cost": 0.0037659959999999998, "time": 111.267901968956}, "d6c4e48eeb": {"quality": 0.21752136752136753, "cost": 0.008788099, "time": 116.89563345909119}, "d6c60a5214": {"quality": 0.17307692307692307, "cost": 0.0012564660000000001, "time": 69.47750086784363}, "d6cbf265ee": {"quality": 0.06538461538461539, "cost": 0.00160378, "time": 72.91289446353912}, "d73a9aab4e": {"quality": 0.04871794871794872, "cost": 0.0032188139999999995, "time": 79.5758284330368}, "d752c30d07": {"quality": 0.3175213675213675, "cost": 0.015617612, "time": 127.57117984294892}, "d7c0972014": {"quality": 0.19444444444444442, "cost": 0.0105163, "time": 80.39488637447357}, "d7f6c0c9d4": {"quality": 0.0, "cost": 0.002123865, "time": 94.49595766067506}, "d813410e44": {"quality": 0.04807692307692308, "cost": 0.002987295, "time": 133.61272318363189}, "d87eb775da": {"quality": 0.2098290598290598, "cost": 0.008013359000000001, "time": 119.17614867687226}, "d8bab6c09b": {"quality": 0.07307692307692308, "cost": 0.011752513, "time": 155.0189305305481}, "d8bcac36e8": {"quality": 0.025, "cost": 0.010723482, "time": 115.70462930202484}, "d96677d8d4": {"quality": 0.21752136752136753, "cost": 0.006440180999999999, "time": 108.42673063278198}, "d9e2bb21a3": {"quality": 0.19594017094017094, "cost": 0.005149365, "time": 107.66626167297363}, "daaadadcc9": {"quality": 0.21752136752136753, "cost": 0.009858946, "time": 142.47570564746857}, "daf855e065": {"quality": 0.1987179487179487, "cost": 0.005835996000000001, "time": 153.1451871395111}, "db6f7259cd": {"quality": 0.17307692307692307, "cost": 0.007230853000000001, "time": 107.38680810928344}, "db9060cd27": {"quality": 0.025, "cost": 0.00183247, "time": 75.05918591022493}, "dbce95a072": {"quality": 0.10641025641025642, "cost": 0.0072439719999999996, "time": 107.68909630775451}, "dbe7d818fa": {"quality": 0.14444444444444443, "cost": 0.0020769660000000004, "time": 109.260413813591}, "dc66bccb1c": {"quality": 0.2098290598290598, "cost": 0.00518188, "time": 101.09860997200012}, "dc90065dea": {"quality": 0.11752136752136753, "cost": 0.010587499, "time": 108.61172733306884}, "dc9a912501": {"quality": 0.06538461538461539, "cost": 0.004132665, "time": 122.22055804729462}, "dd0d70fedd": {"quality": 0.1987179487179487, "cost": 0.0018010649999999997, "time": 72.4727225780487}, "dd76899626": {"quality": 0.0, "cost": 0.01104418, "time": 163.46480329036712}, "dd9f5d1ba9": {"quality": 0.1987179487179487, "cost": 0.003202473, "time": 168.22212414741517}, "de1645053b": {"quality": 0.21752136752136753, "cost": 0.009485828, "time": 156.01319019794465}, "de18bf45e1": {"quality": 0.03333333333333333, "cost": 0.001299202, "time": 79.47014775276185}, "de1e56370f": {"quality": 0.2098290598290598, "cost": 0.0039003330000000006, "time": 95.9440367937088}, "deb84ddd06": {"quality": 0.0, "cost": 0.006051836, "time": 89.6067531824112}, "df2bb408cf": {"quality": 0.125, "cost": 0.0037215599999999996, "time": 133.46323487758636}, "df2ebe2c01": {"quality": 0.1987179487179487, "cost": 0.0045562350000000005, "time": 174.54163272380828}, "dfb8aebe38": {"quality": 0.21752136752136753, "cost": 0.008408229, "time": 166.86121113300322}, "dfce6153aa": {"quality": 0.08482905982905983, "cost": 0.00791217, "time": 165.6665771961212}, "dfda94bd2a": {"quality": 0.0, "cost": 0.0017028, "time": 85.28198800086975}, "dff452a9ca": {"quality": 0.0, "cost": 0.005131782, "time": 116.72450284957885}, "e02f982a26": {"quality": 0.2064102564102564, "cost": 0.00861959, "time": 133.95475313663482}, "e06701b665": {"quality": 0.07222222222222222, "cost": 0.005247648, "time": 155.51543612480162}, "e18abd2ab0": {"quality": 0.19252136752136753, "cost": 0.008510738, "time": 142.51286814212799}, "e1e596ee1b": {"quality": 0.06944444444444445, "cost": 0.009328152, "time": 170.3236501932144}, "e20ba014a1": {"quality": 0.25085470085470085, "cost": 0.010510256999999999, "time": 149.4374349117279}, "e223700849": {"quality": 0.10982905982905983, "cost": 0.002881368, "time": 120.50577642917634}, "e26c7bfbdb": {"quality": 0.08141025641025641, "cost": 0.007298063999999999, "time": 115.83573853969574}, "e35e5f81a7": {"quality": 0.0, "cost": 0.006075344999999999, "time": 117.27980465888976}, "e3cdc0d870": {"quality": 0.09444444444444444, "cost": 0.00339135, "time": 131.6535663843155}, "e3df4cf041": {"quality": 0.06538461538461539, "cost": 0.007586271, "time": 106.25668971538545}, "e3eca9854c": {"quality": 0.10705128205128206, "cost": 0.004585331999999999, "time": 159.58829066753387}, "e47dc3abca": {"quality": 0.09252136752136753, "cost": 0.011292871999999999, "time": 107.18091866970062}, "e495ff601f": {"quality": 0.14807692307692308, "cost": 0.007825082, "time": 156.01348607540132}, "e4b9d4fb41": {"quality": 0.14038461538461539, "cost": 0.007481999, "time": 112.45064301490783}, "e510bda989": {"quality": 0.2098290598290598, "cost": 0.005625576, "time": 35.45525462627411}, "e515bc1935": {"quality": 0.2098290598290598, "cost": 0.00642271, "time": 124.40159273147583}, "e517cd2222": {"quality": 0.1737179487179487, "cost": 0.0015429100000000002, "time": 54.5986225605011}, "e51b01f418": {"quality": 0.1952991452991453, "cost": 0.005621012999999999, "time": 119.1760358095169}, "e520dfae5b": {"quality": 0.2098290598290598, "cost": 0.005809304999999999, "time": 154.96505286693574}, "e53906f84b": {"quality": 0.17649572649572648, "cost": 0.003217785, "time": 119.95617558956147}, "e53b349cce": {"quality": 0.04807692307692308, "cost": 0.006779802, "time": 116.54631357192993}, "e5401ed278": {"quality": 0.1987179487179487, "cost": 0.009414028, "time": 154.65711226463316}, "e56a16ca66": {"quality": 0.3237179487179487, "cost": 0.001822221, "time": 82.61048684120178}, "e5a2f72b30": {"quality": 0.2098290598290598, "cost": 0.008565615, "time": 130.87860839366914}, "e5a70a13ac": {"quality": 0.19252136752136753, "cost": 0.007232057000000002, "time": 115.96633384227752}, "e5fdeb4de9": {"quality": 0.1987179487179487, "cost": 0.0035100910000000003, "time": 106.90362923145295}, "e609601eee": {"quality": 0.13333333333333333, "cost": 0.001958406, "time": 89.7593854188919}, "e6f141cc8f": {"quality": 0.1737179487179487, "cost": 0.007898352, "time": 159.6583716392517}, "e7525c117a": {"quality": 0.125, "cost": 0.003349254, "time": 129.90280907154084}, "e7e94ab7a5": {"quality": 0.1, "cost": 0.001623004, "time": 95.54465639591217}, "e86bd256d6": {"quality": 0.12863247863247865, "cost": 0.003572466, "time": 121.87137434482574}, "e89283a4d9": {"quality": 0.17222222222222222, "cost": 0.0017132999999999998, "time": 125.78474328517913}, "e9befb80e0": {"quality": 0.10982905982905983, "cost": 0.0064075799999999995, "time": 125.61113278865814}, "ea38031fc1": {"quality": 0.08482905982905983, "cost": 0.007648266000000001, "time": 158.89819700717925}, "ea6ecc5653": {"quality": 0.0626068376068376, "cost": 0.007534413, "time": 126.29475154876708}, "ea8bcb3ae2": {"quality": 0.16944444444444445, "cost": 0.01714652, "time": 120.57036209106445}, "ea91a2e78b": {"quality": 0.15, "cost": 0.005214324, "time": 129.3872970342636}, "eac300f0d1": {"quality": 0.1737179487179487, "cost": 0.008836863, "time": 174.77539343833922}, "ebaa9b1297": {"quality": 0.2098290598290598, "cost": 0.001620009, "time": 84.55729002952575}, "ebbe8b6c4f": {"quality": 0.0, "cost": 0.006925624, "time": 89.16193358898164}, "ebdf3abff2": {"quality": 0.10982905982905983, "cost": 0.013354845, "time": 123.62594261169434}, "ebee1f2761": {"quality": 0.22863247863247865, "cost": 0.004503177, "time": 143.60831434726714}, "ec8844a5ae": {"quality": 0.025, "cost": 0.00061108, "time": 93.01526029109955}, "ecb5f78f37": {"quality": 0.0, "cost": 0.010665879, "time": 184.15539872646332}, "ece7ff5129": {"quality": 0.09871794871794873, "cost": 0.006345035999999999, "time": 107.36751945018767}, "ed60b8cac5": {"quality": 0.025, "cost": 0.005388696, "time": 182.1943165063858}, "ed6b5480a5": {"quality": 0.0702991452991453, "cost": 0.004838202, "time": 76.91644642353057}, "eda630dc85": {"quality": 0.21752136752136753, "cost": 0.0075326360000000005, "time": 115.44470648765564}, "edaaee5ed4": {"quality": 0.07307692307692308, "cost": 0.006511264000000001, "time": 83.00507278442383}, "edb2b764aa": {"quality": 0.08482905982905983, "cost": 0.013305772, "time": 130.48116376399994}, "edc52339db": {"quality": 0.04444444444444444, "cost": 0.012820474000000002, "time": 177.77201774120329}, "ee46042c5d": {"quality": 0.14444444444444443, "cost": 0.011933044, "time": 173.3385812520981}, "ee855899d8": {"quality": 0.19444444444444442, "cost": 0.002078877, "time": 131.02240426540374}, "eef12d478b": {"quality": 0.06752136752136753, "cost": 0.005545968, "time": 137.4901770591736}, "ef37b3e0be": {"quality": 0.0, "cost": 0.008727486, "time": 182.45494146347045}, "ef43d497f1": {"quality": 0.13974358974358975, "cost": 0.012247042, "time": 182.13725912570953}, "ef4d4c4a62": {"quality": 0.3175213675213675, "cost": 0.008570482, "time": 138.68434212207794}, "f0655621af": {"quality": 0.05641025641025641, "cost": 0.004645079999999999, "time": 51.96998543739319}, "f076b4c9ae": {"quality": 0.1987179487179487, "cost": 0.008021229, "time": 145.75664427280424}, "f07881a734": {"quality": 0.0, "cost": 0.0008367299999999999, "time": 121.20072700977326}, "f0829510fc": {"quality": 0.12585470085470085, "cost": 0.007723235999999999, "time": 174.65302150249482}, "f11eddb4ed": {"quality": 0.22863247863247865, "cost": 0.011729255, "time": 164.10010514259338}, "f12622d3d7": {"quality": 0.0, "cost": 0.008399868000000001, "time": 176.69494199752808}, "f1408da253": {"quality": 0.2098290598290598, "cost": 0.009230984999999999, "time": 164.57058210372924}, "f1770e7d28": {"quality": 0.2098290598290598, "cost": 0.003513861, "time": 159.09384961128234}, "f18cf41929": {"quality": 0.0, "cost": 0.003205404, "time": 109.51144468784332}, "f1bda127f6": {"quality": 0.21752136752136753, "cost": 0.012127632, "time": 119.53270018100739}, "f2a2e91541": {"quality": 0.1814102564102564, "cost": 0.008741034, "time": 164.1866457939148}, "f2c04ed1c8": {"quality": 0.04038461538461539, "cost": 0.004816446, "time": 76.6830270767212}, "f2cf5db12d": {"quality": 0.09252136752136753, "cost": 0.008174008, "time": 126.23963840007782}, "f3c7a062f7": {"quality": 0.18482905982905984, "cost": 0.003666486, "time": 122.12519326210023}, "f42277df7f": {"quality": 0.2098290598290598, "cost": 0.001627851, "time": 72.52343001365662}, "f437481e3b": {"quality": 0.21752136752136753, "cost": 0.011085886000000001, "time": 157.97356908321382}, "f487340019": {"quality": 0.21816239316239316, "cost": 0.0031079669999999997, "time": 131.89823365211487}, "f497b83523": {"quality": 0.015384615384615385, "cost": 0.007367326, "time": 130.5487345457077}, "f4aa8ffdf5": {"quality": 0.18333333333333332, "cost": 0.00532851, "time": 143.96070950031282}, "f4ab4a73b6": {"quality": 0.07307692307692308, "cost": 0.0073043940000000005, "time": 143.23261981010438}, "f4beb148d0": {"quality": 0.2098290598290598, "cost": 0.0023591489999999996, "time": 131.58125035762788}, "f4deb72db6": {"quality": 0.0, "cost": 0.007903169999999998, "time": 146.17728536129}, "f50a47a0aa": {"quality": 0.15833333333333333, "cost": 0.003946065, "time": 136.4745246887207}, "f5b9a94dcc": {"quality": 0.0, "cost": 0.002980708, "time": 114.85360951423645}, "f5c27e7172": {"quality": 0.015384615384615385, "cost": 0.003205683, "time": 136.12301714420317}, "f5e53d963b": {"quality": 0.07222222222222222, "cost": 0.001996362, "time": 87.49879748821257}, "f5ecac6743": {"quality": 0.23974358974358972, "cost": 0.008653056000000001, "time": 190.25079679489136}, "f614235c15": {"quality": 0.1, "cost": 0.001451158, "time": 103.29747097492219}, "f627bff3a1": {"quality": 0.08333333333333334, "cost": 0.002846523, "time": 154.48251981735228}, "f70e54a9a0": {"quality": 0.2098290598290598, "cost": 0.008556343000000001, "time": 184.88659045696258}, "f74c5a862e": {"quality": 0.0, "cost": 0.006900402, "time": 186.4667979478836}, "f74ec023e4": {"quality": 0.19252136752136753, "cost": 0.011753, "time": 179.87489347457887}, "f783b2b34a": {"quality": 0.1987179487179487, "cost": 0.00525324, "time": 161.1631241083145}, "f7b048bd54": {"quality": 0.19444444444444442, "cost": 0.008355386999999999, "time": 145.28592591285707}, "f7c4df993e": {"quality": 0.2098290598290598, "cost": 0.007574031, "time": 157.67136673927308}, "f848813dca": {"quality": 0.02222222222222222, "cost": 0.0046792710000000005, "time": 184.49361929893493}, "f854533145": {"quality": 0.21752136752136753, "cost": 0.010477562999999999, "time": 188.20525524616244}, "f8f946b5fb": {"quality": 0.032692307692307694, "cost": 0.007302053999999999, "time": 199.7285678625107}, "f912592c8d": {"quality": 0.21752136752136753, "cost": 0.009682198999999999, "time": 187.25646502971648}, "f93d9a2693": {"quality": 0.10705128205128206, "cost": 0.006198954, "time": 124.54699866771698}, "f99096d89c": {"quality": 0.21752136752136753, "cost": 0.008768471, "time": 151.46780948638917}, "f9e8e221f3": {"quality": 0.06944444444444445, "cost": 0.006739169999999999, "time": 118.0668850183487}, "fa38879eab": {"quality": 0.17307692307692307, "cost": 0.013126675, "time": 162.19450600147246}, "fa5b473f15": {"quality": 0.0, "cost": 0.007688484000000001, "time": 179.80013384819028}, "fa7882d46b": {"quality": 0.21752136752136753, "cost": 0.011772778000000001, "time": 164.46603999137878}, "fa906520d1": {"quality": 0.2098290598290598, "cost": 0.012046285, "time": 162.0444113969803}, "faabebaa30": {"quality": 0.21752136752136753, "cost": 0.009497244000000002, "time": 166.34633650779722}, "fb0339a7d0": {"quality": 0.1952991452991453, "cost": 0.012292273000000001, "time": 170.6160633802414}, "fb0e796cd3": {"quality": 0.15149572649572648, "cost": 0.005093987999999999, "time": 133.95573093891142}, "fb216ad6b3": {"quality": 0.04038461538461539, "cost": 0.0032487699999999998, "time": 63.50213906764984}, "fb29372712": {"quality": 0.21752136752136753, "cost": 0.007952629, "time": 162.53553793430328}, "fb6216880a": {"quality": 0.0, "cost": 0.010096947, "time": 167.64952411651612}, "fbd6c45271": {"quality": 0.21752136752136753, "cost": 0.009907777, "time": 157.9287733793259}, "fc1fd5bf54": {"quality": 0.32863247863247863, "cost": 0.0051455070000000006, "time": 150.8760426044464}, "fc4832696b": {"quality": 0.032692307692307694, "cost": 0.001692516, "time": 101.06289932727813}, "fccaadfcdb": {"quality": 0.2098290598290598, "cost": 0.008330596999999999, "time": 168.6835078954697}, "fcd3d2b250": {"quality": 0.18482905982905984, "cost": 0.008048325, "time": 152.26023478507994}, "fce38334b2": {"quality": 0.21752136752136753, "cost": 0.01259885, "time": 138.37961835861205}, "fd0709359e": {"quality": 0.16538461538461538, "cost": 0.0006104649999999999, "time": 45.40659141540527}, "fddccfbf94": {"quality": 0.21752136752136753, "cost": 0.010504784000000001, "time": 108.93906075954438}, "fe2b4d4d8b": {"quality": 0.24316239316239316, "cost": 0.003949674, "time": 128.27600796222686}, "fea4734c09": {"quality": 0.0702991452991453, "cost": 0.0056253, "time": 73.18559403419495}, "fef1ca27fa": {"quality": 0.09252136752136753, "cost": 0.012806324, "time": 116.60554842948915}, "ff11cb6a7a": {"quality": 0.19252136752136753, "cost": 0.012347635999999999, "time": 110.44815881252289}, "ff171e34e2": {"quality": 0.0, "cost": 0.0037192830000000003, "time": 85.79805471897126}, "ff1c958e21": {"quality": 0.0, "cost": 0.002683884, "time": 74.0030293226242}, "ff8e68049a": {"quality": 0.10085470085470086, "cost": 0.006070932, "time": 85.60206592082977}} ================================================ FILE: abacus-research/biodex-priors.json ================================================ {"00c93aec22": {"quality": 0.06301075268817204, "cost": 0.011813586, "time": 87.30672872066498}, "00f4acd0d3": {"quality": 0.0, "cost": 0.0060519300000000005, "time": 43.473920822143555}, "0121878170": {"quality": 0.10301075268817204, "cost": 0.039456927, "time": 69.10648686885834}, "01c2f973ad": {"quality": 0.049677419354838714, "cost": 0.002479065, "time": 42.231227970123285}, "01fca3c717": {"quality": 0.0929032258064516, "cost": 0.029593058000000005, "time": 37.2765305519104}, "02078988c1": {"quality": 0.0, "cost": 0.007589632000000001, "time": 36.742701053619385}, "021604dec1": {"quality": 0.10301075268817204, "cost": 0.009616495, "time": 70.4432460308075}, "0262668df7": {"quality": 0.10301075268817204, "cost": 0.007743969, "time": 52.63532865047455}, "0267c97b70": {"quality": 0.0, "cost": 0.0035319149999999996, "time": 44.35502307415008}, "02d6cdecdc": {"quality": 0.10301075268817204, "cost": 0.019585369999999998, "time": 71.90730912685395}, "033ca325e6": {"quality": 0.03935483870967742, "cost": 0.0024435539999999997, "time": 32.108777427673346}, "0364b5e990": {"quality": 0.06258064516129032, "cost": 0.03804064, "time": 75.88355865478516}, "0375ea52c9": {"quality": 0.12301075268817205, "cost": 0.011332940000000001, "time": 23.44224610328674}, "038a5f0a62": {"quality": 0.10301075268817204, "cost": 0.006738629999999999, "time": 72.20387270450593}, "039803b3b1": {"quality": 0.12301075268817205, "cost": 0.029192550000000005, "time": 28.46143364906311}, "03b972cb56": {"quality": 0.10946236559139785, "cost": 0.024332390000000002, "time": 96.93496084213257}, "042d933706": {"quality": 0.11591397849462365, "cost": 0.008572104, "time": 87.77981555461884}, "050b21ce37": {"quality": 0.10623655913978494, "cost": 0.033586281999999995, "time": 91.73336846828461}, "0524f42520": {"quality": 0.10301075268817204, "cost": 0.04777091600000001, "time": 100.57431967258452}, "05420351e5": {"quality": 0.10301075268817204, "cost": 0.01269706, "time": 76.56351339817047}, "057e332ab1": {"quality": 0.12623655913978493, "cost": 0.03724580200000001, "time": 96.07672710418701}, "0646f3f0fb": {"quality": 0.12301075268817205, "cost": 0.03542941100000001, "time": 98.46862163543702}, "06493715cc": {"quality": 0.0, "cost": 0.005305185, "time": 88.10458414554597}, "0659531b94": {"quality": 0.12623655913978493, "cost": 0.031133855000000005, "time": 76.578067111969}, "067ee6e91b": {"quality": 0.10301075268817204, "cost": 0.007431704999999999, "time": 72.39745917320252}, "0695f9b5fc": {"quality": 0.020046082949308756, "cost": 0.008172494999999998, "time": 89.09118111133574}, "06e94a0f2e": {"quality": 0.10301075268817204, "cost": 0.008446952, "time": 43.131861782073976}, "073ef31d23": {"quality": 0.0929032258064516, "cost": 0.031532903, "time": 66.24685339927674}, "078a7e545e": {"quality": 0.10301075268817204, "cost": 0.031617475000000006, "time": 66.09703652858735}, "079feb14a8": {"quality": 0.11591397849462365, "cost": 0.008705902, "time": 48.95988223552703}, "07a3a7daf7": {"quality": 0.0, "cost": 0.007065148, "time": 43.999537682533266}, "08127cd6dd": {"quality": 0.11268817204301075, "cost": 0.013250458, "time": 49.02173194885254}, "0833133620": {"quality": 0.10623655913978494, "cost": 0.00606747, "time": 52.81210649013519}, "087a2cabc4": {"quality": 0.11591397849462365, "cost": 0.020192525000000003, "time": 94.1397991657257}, "08bf8cc191": {"quality": 0.0, "cost": 0.012546207, "time": 107.96688709259033}, "08e1802287": {"quality": 0.12946236559139784, "cost": 0.0034025700000000006, "time": 25.23361747264862}, "08f7f63b30": {"quality": 0.07290322580645162, "cost": 0.031714827, "time": 104.96963531970978}, "090cd3ef31": {"quality": 0.10946236559139785, "cost": 0.00352441, "time": 23.527972793579103}, "0947216ece": {"quality": 0.11268817204301075, "cost": 0.014792812, "time": 80.49954354763031}, "096d51f670": {"quality": 0.08301075268817204, "cost": 0.009779318999999998, "time": 120.63433356285095}, "09791c731b": {"quality": 0.08623655913978495, "cost": 0.007008072, "time": 91.62384450435638}, "0990c0d4f8": {"quality": 0.10301075268817204, "cost": 0.007845877, "time": 85.36602926254272}, "0a4c1bbb4a": {"quality": 0.0, "cost": 0.0061861469999999995, "time": 90.79566612243653}, "0ac969dde3": {"quality": 0.10301075268817204, "cost": 0.010119635, "time": 114.36860978603363}, "0b4ab72197": {"quality": 0.08301075268817204, "cost": 0.010272084, "time": 85.533615732193}, "0bf9d31691": {"quality": 0.0, "cost": 0.04230826, "time": 110.75088348388672}, "0c020b86a3": {"quality": 0.06, "cost": 0.009179564, "time": 80.5141788482666}, "0c6c7fe96a": {"quality": 0.11591397849462365, "cost": 0.012506979000000001, "time": 104.92288007736207}, "0c81c8996a": {"quality": 0.03333333333333333, "cost": 0.006368825999999999, "time": 67.1365476846695}, "0cd25da9fe": {"quality": 0.1429032258064516, "cost": 0.037754836, "time": 88.54509041309356}, "0cd78f33d8": {"quality": 0.12301075268817205, "cost": 0.05730706500000001, "time": 70.51942982673646}, "0cdc5954dd": {"quality": 0.06623655913978495, "cost": 0.008238566000000001, "time": 70.26068692207338}, "0d53dd53c1": {"quality": 0.10301075268817204, "cost": 0.040789094, "time": 83.40459337234498}, "0d8436af32": {"quality": 0.10301075268817204, "cost": 0.011668765, "time": 70.86097333431243}, "0e38896654": {"quality": 0.12301075268817205, "cost": 0.011624445, "time": 69.54783916473389}, "0ec672e7c8": {"quality": 0.10301075268817204, "cost": 0.013338835, "time": 92.46337025165558}, "0ed243f788": {"quality": 0.11591397849462365, "cost": 0.009052659000000001, "time": 82.04982092380524}, "0eeb372802": {"quality": 0.08489247311827958, "cost": 0.01393459, "time": 87.2686581134796}, "0ef0becc1b": {"quality": 0.14562980030721967, "cost": 0.028691548000000004, "time": 57.96495416164398}, "0fefead197": {"quality": 0.10623655913978494, "cost": 0.036967563, "time": 86.80253887176514}, "0ff126ebf8": {"quality": 0.10623655913978494, "cost": 0.012027738000000001, "time": 62.63344478607178}, "10d1d4bdeb": {"quality": 0.0, "cost": 0.034578758, "time": 84.60008835792542}, "114a097c53": {"quality": 0.02967741935483871, "cost": 0.00546162, "time": 32.99331085681915}, "1175ee37e6": {"quality": 0.06301075268817204, "cost": 0.0015683849999999998, "time": 27.241142559051514}, "11bc996d48": {"quality": 0.10301075268817204, "cost": 0.006689505, "time": 67.14279909133911}, "11ded03305": {"quality": 0.12301075268817205, "cost": 0.03211208700000001, "time": 71.28769545555114}, "123fb650fb": {"quality": 0.0704147465437788, "cost": 0.009157566, "time": 94.18252191543579}, "132f6f3946": {"quality": 0.0, "cost": 0.034120617000000006, "time": 80.40761358737946}, "133ee5023f": {"quality": 0.08301075268817204, "cost": 0.0056809600000000005, "time": 41.6411509513855}, "13a009fe0c": {"quality": 0.12290322580645162, "cost": 0.013064816, "time": 103.28004715442657}, "13da306f84": {"quality": 0.10301075268817204, "cost": 0.009303667999999998, "time": 75.9394079208374}, "13e717e41e": {"quality": 0.10301075268817204, "cost": 0.027711322000000004, "time": 46.58983483314514}, "13f75f9bd0": {"quality": 0.06301075268817204, "cost": 0.007269792000000001, "time": 78.83892266750337}, "140ededb41": {"quality": 0.07290322580645162, "cost": 0.0016495350000000002, "time": 27.533066105842593}, "142e59c03f": {"quality": 0.10301075268817204, "cost": 0.051444216, "time": 89.78037304878235}, "142f3a7c70": {"quality": 0.0, "cost": 0.0048542970000000005, "time": 90.07974328994752}, "1468dddecc": {"quality": 0.12623655913978493, "cost": 0.0025156650000000003, "time": 58.17216382026672}, "15af009a01": {"quality": 0.09333333333333334, "cost": 0.03807909200000001, "time": 106.9638382911682}, "15b80a55d3": {"quality": 0.09333333333333334, "cost": 0.036063054, "time": 102.59992785453797}, "1625e624c5": {"quality": 0.0696774193548387, "cost": 0.02807447, "time": 56.73278093338013}, "16cff1c1e9": {"quality": 0.0, "cost": 0.031882792, "time": 110.76414783000945}, "17407df027": {"quality": 0.08623655913978495, "cost": 0.035592325999999994, "time": 101.71423263549804}, "176da24f53": {"quality": 0.014285714285714285, "cost": 0.004459658, "time": 50.037702202796936}, "179379555f": {"quality": 0.08946236559139785, "cost": 0.006388011, "time": 77.25874888896942}, "181c91d1be": {"quality": 0.10623655913978494, "cost": 0.012226980000000002, "time": 70.32721478939057}, "183743e76e": {"quality": 0.0, "cost": 0.00601107, "time": 54.00228130817413}, "186b58c209": {"quality": 0.10623655913978494, "cost": 0.03340172, "time": 77.13717110157013}, "187eace9fe": {"quality": 0.0, "cost": 0.031072944, "time": 83.48713932037353}, "190ed2e1b6": {"quality": 0.20301075268817204, "cost": 0.032216430000000004, "time": 60.7668753862381}, "191aafe1a6": {"quality": 0.10301075268817204, "cost": 0.028626901, "time": 69.7408289194107}, "194919ad28": {"quality": 0.032903225806451615, "cost": 0.032168366000000004, "time": 64.83868379592896}, "197bb53f10": {"quality": 0.09333333333333334, "cost": 0.0064794810000000005, "time": 85.31207880973815}, "19ba7d0617": {"quality": 0.12301075268817205, "cost": 0.0850125, "time": 65.46971595287323}, "19e3db7fe7": {"quality": 0.03333333333333333, "cost": 0.012209139999999999, "time": 111.22203342914581}, "1a08cb3f50": {"quality": 0.0, "cost": 0.031574925000000004, "time": 48.5790287733078}, "1a169179f6": {"quality": 0.10301075268817204, "cost": 0.044995354, "time": 103.08906679153444}, "1a71d61ac4": {"quality": 0.08301075268817204, "cost": 0.02794963, "time": 76.25873317718506}, "1ad856985f": {"quality": 0.02258064516129032, "cost": 0.009150617, "time": 122.38001575469971}, "1adec2dca2": {"quality": 0.0, "cost": 0.01436658, "time": 146.7017418861389}, "1b04a2b184": {"quality": 0.11591397849462365, "cost": 0.013684314999999999, "time": 133.7267296552658}, "1b28439bd7": {"quality": 0.21591397849462365, "cost": 0.011476702000000002, "time": 100.34343218803406}, "1b2c667b15": {"quality": 0.19333333333333333, "cost": 0.033035388, "time": 89.06020038127899}, "1b4511eada": {"quality": 0.11456989247311827, "cost": 0.030632538, "time": 129.76828122138977}, "1b7e6cad66": {"quality": 0.10301075268817204, "cost": 0.028176118, "time": 68.36105287075043}, "1beb2fac62": {"quality": 0.10301075268817204, "cost": 0.013600296000000001, "time": 84.02366802692413}, "1c347e4d91": {"quality": 0.10301075268817204, "cost": 0.003799887, "time": 104.80383982658387}, "1c35bf4be6": {"quality": 0.10623655913978494, "cost": 0.03395996, "time": 116.73827676773071}, "1c4bbf8f7e": {"quality": 0.10301075268817204, "cost": 0.04330549100000001, "time": 104.88216242790222}, "1c5f1341f6": {"quality": 0.10623655913978494, "cost": 0.023331052, "time": 78.7630702495575}, "1c71804bec": {"quality": 0.08967741935483872, "cost": 0.019993577999999998, "time": 79.78316872119903}, "1cc6d9efb6": {"quality": 0.10301075268817204, "cost": 0.012043020000000002, "time": 78.92661263942719}, "1ce3d77039": {"quality": 0.10301075268817204, "cost": 0.000653472, "time": 41.55318109989166}, "1d26090364": {"quality": 0.12301075268817205, "cost": 0.004974435, "time": 56.773493957519534}, "1d87f97e62": {"quality": 0.10301075268817204, "cost": 0.007740711000000001, "time": 77.09953954219819}, "1d90fb8ca6": {"quality": 0.10623655913978494, "cost": 0.034693865000000004, "time": 72.57712695598602}, "1da2369719": {"quality": 0.032903225806451615, "cost": 0.004285659000000001, "time": 71.20325164794922}, "1e18e60895": {"quality": 0.12301075268817205, "cost": 0.0006631269999999999, "time": 39.11586356163025}, "1e1bf7e88b": {"quality": 0.10623655913978494, "cost": 0.012434028, "time": 72.40708291530609}, "1e8b3521f8": {"quality": 0.07729646697388634, "cost": 0.003250434, "time": 60.45711851119995}, "1f412964ff": {"quality": 0.12301075268817205, "cost": 0.029133057000000004, "time": 64.97693083286285}, "1f72cfb78a": {"quality": 0.21591397849462365, "cost": 0.020125592, "time": 99.64003388881683}, "1fb5d170ad": {"quality": 0.11268817204301075, "cost": 0.031407344000000004, "time": 107.89154133796691}, "20180dd292": {"quality": 0.03333333333333333, "cost": 0.035306141, "time": 115.35444395542144}, "2018bef45f": {"quality": 0.10301075268817204, "cost": 0.007217930000000001, "time": 68.45855422019959}, "2075ff1d04": {"quality": 0.06, "cost": 0.000997618, "time": 56.56468660831451}, "208a98f514": {"quality": 0.06623655913978495, "cost": 0.010588595, "time": 121.87733623981475}, "20904e5c14": {"quality": 0.08967741935483872, "cost": 0.031855074, "time": 83.47015318870544}, "20afc3d539": {"quality": 0.23591397849462367, "cost": 0.039295448, "time": 91.55873837471009}, "20e10af7d4": {"quality": 0.09591397849462366, "cost": 0.005751711, "time": 109.45092914104461}, "20e2c0b057": {"quality": 0.04, "cost": 0.008348217000000002, "time": 118.59764387607575}, "211b89b4cd": {"quality": 0.10301075268817204, "cost": 0.005769704999999999, "time": 68.7529235124588}, "21386082aa": {"quality": 0.10946236559139785, "cost": 0.019664704, "time": 109.10838623046875}, "21b2b8ebd1": {"quality": 0.10623655913978494, "cost": 0.0073147049999999995, "time": 87.6013917684555}, "21b78249a7": {"quality": 0.10301075268817204, "cost": 0.04023389200000001, "time": 85.05358052253723}, "21bed16a7d": {"quality": 0.05290322580645161, "cost": 0.0014240299999999997, "time": 44.216261696815494}, "2200d969d0": {"quality": 0.10301075268817204, "cost": 0.042653065000000004, "time": 104.29301726818085}, "220d008704": {"quality": 0.02, "cost": 0.030364681, "time": 107.97830784320831}, "2251d21392": {"quality": 0.11591397849462365, "cost": 0.033143465, "time": 79.32752299308777}, "23566f15ab": {"quality": 0.08301075268817204, "cost": 0.0005060469999999999, "time": 43.435147762298584}, "23a9506d36": {"quality": 0.08301075268817204, "cost": 0.005874952, "time": 36.02558331489563}, "24957f3a43": {"quality": 0.08946236559139785, "cost": 0.004644396, "time": 53.15490672588348}, "2529e2f8b0": {"quality": 0.10301075268817204, "cost": 0.007180190000000001, "time": 28.930508136749268}, "252f01ac5b": {"quality": 0.12623655913978493, "cost": 0.04125042200000001, "time": 92.3930985212326}, "25fadf0883": {"quality": 0.10946236559139785, "cost": 0.03512302, "time": 116.36114256381988}, "2609bfd616": {"quality": 0.10946236559139785, "cost": 0.009596338, "time": 116.62334115505219}, "2629f3e324": {"quality": 0.12301075268817205, "cost": 0.033418977, "time": 125.81296608448028}, "262e4298f9": {"quality": 0.06301075268817204, "cost": 0.03408896, "time": 126.59840724468232}, "26cc40d3bb": {"quality": 0.06258064516129032, "cost": 0.035955086, "time": 118.60315663814545}, "2728c8eb6a": {"quality": 0.063963133640553, "cost": 0.008921946, "time": 100.46461434364319}, "27bc52befa": {"quality": 0.08623655913978495, "cost": 0.0216924, "time": 65.37463784217834}, "27daa50458": {"quality": 0.19978494623655915, "cost": 0.005977634, "time": 62.87260589599609}, "2821795e69": {"quality": 0.11591397849462365, "cost": 0.035850537, "time": 122.94275135993958}, "28369b2421": {"quality": 0.11591397849462365, "cost": 0.008891583000000002, "time": 96.52785930633544}, "28421e6d62": {"quality": 0.07967741935483871, "cost": 0.007445298000000001, "time": 90.94689693450928}, "2936c3e43e": {"quality": 0.10623655913978494, "cost": 0.012975128999999998, "time": 113.21974639892579}, "293ec5edca": {"quality": 0.10301075268817204, "cost": 0.006139281, "time": 76.99833080768585}, "29409d0894": {"quality": 0.08623655913978495, "cost": 0.005303836, "time": 53.642726635932924}, "294258298a": {"quality": 0.08301075268817204, "cost": 0.029248269, "time": 73.05858094692229}, "294e541235": {"quality": 0.014285714285714285, "cost": 0.002588631, "time": 46.93031387329101}, "295ed5e759": {"quality": 0.03333333333333333, "cost": 0.007936986, "time": 78.30404438972474}, "2960431101": {"quality": 0.07913978494623655, "cost": 0.007295059999999999, "time": 65.45906202793121}, "29892d8468": {"quality": 0.0, "cost": 0.012367010000000001, "time": 107.3028032541275}, "299a0aeb65": {"quality": 0.10301075268817204, "cost": 0.008972781999999999, "time": 99.65134017467498}, "29ad99e3ed": {"quality": 0.012903225806451613, "cost": 0.028833116000000002, "time": 76.06020951271057}, "2a5edac2de": {"quality": 0.10301075268817204, "cost": 0.040388614, "time": 98.854678440094}, "2a7d15f4a7": {"quality": 0.10301075268817204, "cost": 0.0007995049999999998, "time": 26.340464663505553}, "2aa996de6a": {"quality": 0.13591397849462367, "cost": 0.041571458000000006, "time": 101.75389010906218}, "2ac4fb293f": {"quality": 0.10301075268817204, "cost": 0.027978918000000002, "time": 79.7981543302536}, "2afeff0083": {"quality": 0.13134408602150538, "cost": 0.034197613, "time": 108.6436208486557}, "2b2bc9568b": {"quality": 0.12301075268817205, "cost": 0.034515569, "time": 122.13092226982117}, "2b5679d248": {"quality": 0.07591397849462365, "cost": 0.012507200999999999, "time": 111.41059238910675}, "2bcf54cda1": {"quality": 0.05333333333333333, "cost": 0.03352914, "time": 122.20578529834748}, "2bd39ee744": {"quality": 0.10301075268817204, "cost": 0.016464735, "time": 93.0221663236618}, "2bf38d797f": {"quality": 0.09655913978494624, "cost": 0.007761932999999999, "time": 85.16239807605743}, "2c4f4f304e": {"quality": 0.12301075268817205, "cost": 0.004792675, "time": 50.257468938827515}, "2c5cf9eb26": {"quality": 0.0, "cost": 0.006811359, "time": 85.30586485862732}, "2c9a9f94c4": {"quality": 0.06623655913978495, "cost": 0.011060496, "time": 127.40647723674773}, "2d3bbc2d23": {"quality": 0.11333333333333334, "cost": 0.006915585, "time": 98.87370185852052}, "2de113167b": {"quality": 0.20623655913978495, "cost": 0.0120975, "time": 114.90267214775085}, "2de3eb2c19": {"quality": 0.08258064516129032, "cost": 0.006905800000000001, "time": 56.68419787883758}, "2e30394ac6": {"quality": 0.09333333333333334, "cost": 0.007497251999999999, "time": 99.04903969764709}, "2e9c5cc9bf": {"quality": 0.05290322580645161, "cost": 0.006026862, "time": 64.40614473819733}, "2f1573da80": {"quality": 0.0696774193548387, "cost": 0.009294684000000001, "time": 118.76816306114196}, "2f39d78f34": {"quality": 0.12623655913978493, "cost": 0.03424951, "time": 84.87492218017579}, "2fc0cb3592": {"quality": 0.07290322580645162, "cost": 0.007008576, "time": 93.76994898319245}, "2fd9cd426a": {"quality": 0.08301075268817204, "cost": 0.012578212000000002, "time": 125.21965701580046}, "300924ebae": {"quality": 0.10301075268817204, "cost": 0.034383670000000005, "time": 79.05412302017211}, "3019af79b3": {"quality": 0.05333333333333333, "cost": 0.000728678, "time": 50.62756032943726}, "302c1d97fc": {"quality": 0.032903225806451615, "cost": 0.012643149999999999, "time": 94.44454934597016}, "30ae4cbe91": {"quality": 0.04967741935483871, "cost": 0.0006998079999999999, "time": 55.2713984966278}, "30c1f9ddf1": {"quality": 0.02, "cost": 0.031684431000000006, "time": 81.2769385099411}, "30cd375570": {"quality": 0.10623655913978494, "cost": 0.034244302000000004, "time": 84.5775047302246}, "3169782cbb": {"quality": 0.12623655913978493, "cost": 0.030658732000000005, "time": 82.21524584293365}, "3172fc459a": {"quality": 0.20623655913978495, "cost": 0.039465366, "time": 106.1840931892395}, "318499c14b": {"quality": 0.06623655913978495, "cost": 0.03394029, "time": 109.55087904930114}, "31a32be94d": {"quality": 0.12623655913978493, "cost": 0.040973235, "time": 113.42904851436614}, "32b101d807": {"quality": 0.11134408602150538, "cost": 0.013450305, "time": 116.4323141336441}, "32e2c7ad7f": {"quality": 0.10623655913978494, "cost": 0.034506176, "time": 105.92563235759735}, "33459cd29c": {"quality": 0.05333333333333333, "cost": 0.009939888, "time": 108.05581707954407}, "33a187e74f": {"quality": 0.10301075268817204, "cost": 0.00615336, "time": 108.04435174465179}, "33bab4f766": {"quality": 0.10946236559139785, "cost": 0.030359016, "time": 94.6774396419525}, "34922140da": {"quality": 0.12623655913978493, "cost": 0.037889484, "time": 97.54083952903747}, "3511b5e1d0": {"quality": 0.0, "cost": 0.009048384, "time": 107.80388951301575}, "3513311c2d": {"quality": 0.10946236559139785, "cost": 0.030197856000000002, "time": 70.30446681976318}, "3513e54767": {"quality": 0.1529032258064516, "cost": 0.006643148000000001, "time": 51.87993569374085}, "353f0cb1ac": {"quality": 0.07591397849462365, "cost": 0.002595798, "time": 43.97162518501281}, "3550bf88cb": {"quality": 0.10301075268817204, "cost": 0.008131596, "time": 101.78304505348206}, "35610fb420": {"quality": 0.10301075268817204, "cost": 0.004529697, "time": 86.15702996253967}, "357267e14b": {"quality": 0.10301075268817204, "cost": 0.001159102, "time": 43.801218032836914}, "35baa5c3cc": {"quality": 0.12623655913978493, "cost": 0.032426465, "time": 97.53744850158691}, "3637084f91": {"quality": 0.11913978494623656, "cost": 0.00731297, "time": 75.00282621383667}, "368a497102": {"quality": 0.08946236559139785, "cost": 0.023536822000000006, "time": 74.35990719795228}, "36c66671ee": {"quality": 0.049677419354838714, "cost": 0.0066839249999999985, "time": 80.50561456680299}, "37456cb002": {"quality": 0.10623655913978494, "cost": 0.018729613, "time": 104.19728450775146}, "3746ea5c03": {"quality": 0.08946236559139785, "cost": 0.025954912000000004, "time": 77.87212231159211}, "375ed248fe": {"quality": 0.09333333333333334, "cost": 0.009719407999999999, "time": 83.5635404586792}, "377cdf9209": {"quality": 0.10301075268817204, "cost": 0.015890171, "time": 106.57615406513214}, "37d4d0f214": {"quality": 0.09913978494623656, "cost": 0.035574514, "time": 106.92880241870881}, "37ece7217f": {"quality": 0.11729646697388633, "cost": 0.027270672000000003, "time": 49.32679312229156}, "38075bb01f": {"quality": 0.05333333333333333, "cost": 0.0043161810000000005, "time": 80.82736201286316}, "3831d758b1": {"quality": 0.0, "cost": 0.032444862000000005, "time": 82.06478433609009}, "38567d6a43": {"quality": 0.0, "cost": 0.004685955, "time": 89.93808691501617}, "3875787727": {"quality": 0.08301075268817204, "cost": 0.034211713000000005, "time": 105.70414273738861}, "389c54cbca": {"quality": 0.09655913978494624, "cost": 0.00570144, "time": 52.78039865493774}, "3980f20caa": {"quality": 0.05290322580645161, "cost": 0.014630126000000002, "time": 109.6514874458313}, "3997a836bd": {"quality": 0.10301075268817204, "cost": 0.034291603000000004, "time": 108.07982413768768}, "39ad76f8ce": {"quality": 0.10301075268817204, "cost": 0.033542926, "time": 88.07275912761688}, "39c0b7c171": {"quality": 0.10301075268817204, "cost": 0.036100823000000004, "time": 108.34460818767548}, "39cd4ca402": {"quality": 0.02, "cost": 0.0034239629999999995, "time": 84.85748331546785}, "3a32c98a53": {"quality": 0.10623655913978494, "cost": 0.030208363000000002, "time": 82.875235414505}, "3ac7fa4e46": {"quality": 0.10301075268817204, "cost": 0.04524012, "time": 97.64954562187195}, "3ad6dcf559": {"quality": 0.0, "cost": 0.031851822, "time": 80.59675960540771}, "3ae0de8663": {"quality": 0.10301075268817204, "cost": 0.021463644000000004, "time": 97.35158727169036}, "3b2e8075ea": {"quality": 0.08623655913978495, "cost": 0.003255993, "time": 67.06967768669128}, "3b3676521a": {"quality": 0.10301075268817204, "cost": 0.013791060999999999, "time": 95.51536769866944}, "3b57530a56": {"quality": 0.10301075268817204, "cost": 0.002789373, "time": 44.8931683063507}, "3b6fbfa11d": {"quality": 0.10946236559139785, "cost": 0.016405658, "time": 70.66574103832244}, "3b81215e7a": {"quality": 0.11268817204301075, "cost": 0.034937565000000004, "time": 111.87461152076722}, "3b9f8045d7": {"quality": 0.11268817204301075, "cost": 0.044838478, "time": 104.71198470592498}, "3c5857683c": {"quality": 0.10946236559139785, "cost": 0.03896547, "time": 133.13998155593873}, "3cbab8082e": {"quality": 0.09333333333333334, "cost": 0.009450357, "time": 131.19907307624817}, "3d21104666": {"quality": 0.12301075268817205, "cost": 0.05427800000000001, "time": 58.033410573005675}, "3d71c4dd2c": {"quality": 0.10301075268817204, "cost": 0.006745917, "time": 99.85713205337524}, "3d9e24215e": {"quality": 0.10301075268817204, "cost": 0.008923918, "time": 54.70108168125152}, "3e7efee65a": {"quality": 0.09268817204301075, "cost": 0.034440120000000005, "time": 125.9171290397644}, "3ed0ad20ed": {"quality": 0.12301075268817205, "cost": 0.038282694000000006, "time": 132.21433353424072}, "3f1a58aec9": {"quality": 0.22301075268817205, "cost": 0.0019911300000000002, "time": 48.89557406902313}, "3f2b07cb78": {"quality": 0.10946236559139785, "cost": 0.016589166000000002, "time": 105.48563659191132}, "3f3ef494b0": {"quality": 0.0, "cost": 0.001254132, "time": 57.60023159980774}, "3f62c3fbfc": {"quality": 0.0696774193548387, "cost": 0.0044856359999999994, "time": 95.2896116256714}, "3f730d8bfe": {"quality": 0.10301075268817204, "cost": 0.04548097400000001, "time": 94.35254197120668}, "3f8d2ee81f": {"quality": 0.10623655913978494, "cost": 0.02839726, "time": 88.55384962558746}, "40104c813f": {"quality": 0.00967741935483871, "cost": 0.010927247000000001, "time": 137.43559978008273}, "403b05da2d": {"quality": 0.02, "cost": 0.010782219999999999, "time": 89.91662650108339}, "403f0726fa": {"quality": 0.10301075268817204, "cost": 0.030024500000000003, "time": 53.93264591693878}, "4098178354": {"quality": 0.10623655913978494, "cost": 0.033845859000000006, "time": 113.72850313186646}, "409ff67607": {"quality": 0.06, "cost": 0.004197711, "time": 99.27511491775513}, "40b3b6642c": {"quality": 0.12301075268817205, "cost": 0.033175452, "time": 78.3308812379837}, "412c065b83": {"quality": 0.03333333333333333, "cost": 0.00797817, "time": 76.86819369792937}, "4191118787": {"quality": 0.10301075268817204, "cost": 0.009176240999999998, "time": 115.17749345302582}, "41d5b97871": {"quality": 0.0, "cost": 0.002632212, "time": 67.86801817417145}, "41d8845655": {"quality": 0.12301075268817205, "cost": 0.011058695, "time": 66.63209574222564}, "41ee202cac": {"quality": 0.04, "cost": 0.0015582279999999996, "time": 81.97960751056671}, "41fe4aee55": {"quality": 0.04, "cost": 0.033214178, "time": 89.46116530895233}, "42430ea391": {"quality": 0.10301075268817204, "cost": 0.037945212000000006, "time": 121.86067166328431}, "42ddd48341": {"quality": 0.10301075268817204, "cost": 0.015020527999999998, "time": 114.5060165643692}, "42f1e19aa7": {"quality": 0.11591397849462365, "cost": 0.04674747600000001, "time": 126.75622403621674}, "430a2ab32f": {"quality": 0.12301075268817205, "cost": 0.039291168, "time": 96.79555187225341}, "4339427ad8": {"quality": 0.08634408602150537, "cost": 0.03448633, "time": 124.03267226219177}, "4361bc7ea7": {"quality": 0.07333333333333333, "cost": 0.001487792, "time": 83.36836609840392}, "43c3cf9cb8": {"quality": 0.10301075268817204, "cost": 0.006907384000000001, "time": 57.54507291316986}, "43d24fb32a": {"quality": 0.10946236559139785, "cost": 0.010483464, "time": 98.36521954536438}, "43e9b39e5c": {"quality": 0.12946236559139784, "cost": 0.036367179, "time": 118.90290827751159}, "44d6af5523": {"quality": 0.10301075268817204, "cost": 0.0051817799999999995, "time": 97.96241414546967}, "44f189d813": {"quality": 0.10623655913978494, "cost": 0.020515107000000005, "time": 121.6331589460373}, "450f45a187": {"quality": 0.0, "cost": 0.007900044000000002, "time": 60.44815545082092}, "453d0a5097": {"quality": 0.06301075268817204, "cost": 0.016095624, "time": 92.88813369274139}, "4547ef4c8e": {"quality": 0.10301075268817204, "cost": 0.033263573000000005, "time": 89.95854828357696}, "461846a52d": {"quality": 0.10623655913978494, "cost": 0.003919338, "time": 85.1272670507431}, "462e6ff849": {"quality": 0.06761904761904762, "cost": 0.011236148000000001, "time": 113.67141807079315}, "4630853d32": {"quality": 0.08946236559139785, "cost": 0.008533141000000001, "time": 86.52934875488282}, "46475b9e75": {"quality": 0.10301075268817204, "cost": 0.006695604000000001, "time": 94.7075261592865}, "46654a1f32": {"quality": 0.0, "cost": 0.012072104, "time": 64.53883934020996}, "466a3036b2": {"quality": 0.06301075268817204, "cost": 0.010685084999999999, "time": 106.71569502353668}, "466d4d16dd": {"quality": 0.08946236559139785, "cost": 0.008533694000000001, "time": 79.9828891992569}, "46ed68152d": {"quality": 0.07612903225806453, "cost": 0.012683225999999999, "time": 86.68546965122223}, "476a12876c": {"quality": 0.08946236559139785, "cost": 0.0062402939999999995, "time": 72.02064683437348}, "4778401a7a": {"quality": 0.10301075268817204, "cost": 0.016234258, "time": 93.26795687675477}, "47f9115b26": {"quality": 0.03333333333333333, "cost": 0.03462185200000001, "time": 101.22674057483673}, "48043e2304": {"quality": 0.10301075268817204, "cost": 0.009490545, "time": 83.147008061409}, "487f30e740": {"quality": 0.10623655913978494, "cost": 0.030731074000000004, "time": 72.48842267990112}, "488645cbd9": {"quality": 0.07290322580645162, "cost": 0.0015898319999999998, "time": 47.430080437660216}, "48bf87f7fe": {"quality": 0.11591397849462365, "cost": 0.041125237999999995, "time": 95.13780512809754}, "4909061216": {"quality": 0.08301075268817204, "cost": 0.013231131, "time": 110.22792162895203}, "49731b1ccd": {"quality": 0.08301075268817204, "cost": 0.007518614999999999, "time": 77.45786833763123}, "49ad844bd2": {"quality": 0.08301075268817204, "cost": 0.009246200999999999, "time": 125.66613755226135}, "49ca727e49": {"quality": 0.10301075268817204, "cost": 0.000910932, "time": 36.98862104415893}, "4a23d8eff7": {"quality": 0.012903225806451613, "cost": 0.033723964, "time": 133.58634560108186}, "4a555da784": {"quality": 0.12301075268817205, "cost": 0.030253567000000002, "time": 93.47855880260468}, "4a5cea8b85": {"quality": 0.10301075268817204, "cost": 0.037096130000000005, "time": 87.66496036052703}, "4a767339bd": {"quality": 0.10946236559139785, "cost": 0.034482308, "time": 119.38791158199311}, "4aafd39d76": {"quality": 0.0, "cost": 0.005941872, "time": 106.78513460159303}, "4aca6e5216": {"quality": 0.08623655913978495, "cost": 0.011614728, "time": 98.73750612735748}, "4b18a647d6": {"quality": 0.10623655913978494, "cost": 0.04434360500000001, "time": 133.19987111091615}, "4bc4528402": {"quality": 0.10623655913978494, "cost": 0.004866044999999999, "time": 105.87921042442322}, "4c158a1a4a": {"quality": 0.06301075268817204, "cost": 0.008219039999999999, "time": 107.04837877750397}, "4c954323e3": {"quality": 0.11591397849462365, "cost": 0.010109752999999999, "time": 127.91732285022735}, "4d91e8a27b": {"quality": 0.1761290322580645, "cost": 0.010757003000000001, "time": 101.67963089942933}, "4dc185389a": {"quality": 0.08290322580645162, "cost": 0.030485803000000006, "time": 113.32033824920654}, "4dd3635bc3": {"quality": 0.12623655913978493, "cost": 0.013448202, "time": 89.75064220428467}, "4dd98ef398": {"quality": 0.0, "cost": 0.030016080000000004, "time": 69.32772116661073}, "4dfacd0007": {"quality": 0.10623655913978494, "cost": 0.014845295, "time": 100.57037975788117}, "4e298ee0d4": {"quality": 0.07612903225806453, "cost": 0.008624916, "time": 97.4965528011322}, "4e3443a0f9": {"quality": 0.11591397849462365, "cost": 0.013369873000000001, "time": 110.12258355617523}, "4e4b9db2b8": {"quality": 0.116605222734255, "cost": 0.007327941000000001, "time": 75.85998649597168}, "4e6509f614": {"quality": 0.08301075268817204, "cost": 0.00546564, "time": 51.5486006975174}, "4e6a83e751": {"quality": 0.07290322580645162, "cost": 0.000857376, "time": 45.70523474216461}, "4e79c8947f": {"quality": 0.11134408602150539, "cost": 0.00732088, "time": 43.410219454765326}, "4e9504432b": {"quality": 0.10623655913978494, "cost": 0.034706275, "time": 129.37581310272216}, "4e962170dc": {"quality": 0.02, "cost": 0.033169164, "time": 83.41308994293212}, "4eb0826f21": {"quality": 0.0, "cost": 0.027521320000000002, "time": 56.27700278759002}, "4ed41bf2e4": {"quality": 0.12623655913978493, "cost": 0.03861573, "time": 115.21599047183992}, "4f78672528": {"quality": 0.08301075268817204, "cost": 0.037576234, "time": 94.65013363361359}, "4f8cca1195": {"quality": 0.13019969278033794, "cost": 0.010837381, "time": 124.13806178569794}, "500860eaa2": {"quality": 0.0, "cost": 0.002532492, "time": 48.11276173591614}, "50701b505e": {"quality": 0.09333333333333332, "cost": 0.006791684999999999, "time": 99.08809175491334}, "51583a901c": {"quality": 0.14946236559139786, "cost": 0.010773869, "time": 124.03754835128784}, "51aeaf9f3e": {"quality": 0.10301075268817204, "cost": 0.008776062000000001, "time": 40.02036185264588}, "520b52b64c": {"quality": 0.10301075268817204, "cost": 0.0302065, "time": 60.96432254314423}, "521314dab6": {"quality": 0.20301075268817204, "cost": 0.006639104, "time": 69.5313068151474}, "5226eb7ff6": {"quality": 0.10946236559139785, "cost": 0.008551829, "time": 88.08115694522857}, "526878b5eb": {"quality": 0.05290322580645161, "cost": 0.009073641, "time": 127.54378099441527}, "52c1cba6ce": {"quality": 0.11591397849462365, "cost": 0.010276206999999999, "time": 117.55816102027893}, "52f041a70e": {"quality": 0.10301075268817204, "cost": 0.001223052, "time": 42.505560874938965}, "533867574b": {"quality": 0.12301075268817205, "cost": 0.031638318000000006, "time": 88.52275938987732}, "53869388bb": {"quality": 0.05655913978494624, "cost": 0.0018472099999999997, "time": 38.346792578697205}, "53aefd41e4": {"quality": 0.10301075268817204, "cost": 0.022357914, "time": 79.89788761138917}, "53d2932c4f": {"quality": 0.12374807987711213, "cost": 0.007490942, "time": 79.30699775218963}, "54375d3eba": {"quality": 0.04967741935483871, "cost": 0.013380047999999999, "time": 77.51320762634276}, "5474247f91": {"quality": 0.0, "cost": 0.008527589999999998, "time": 80.76838719844818}, "54993bc472": {"quality": 0.13134408602150538, "cost": 0.027307650000000003, "time": 45.53245575428009}, "55358f2285": {"quality": 0.11591397849462365, "cost": 0.034442846, "time": 74.98782951831818}, "5569b4f878": {"quality": 0.10623655913978494, "cost": 0.004829285, "time": 52.80584018230438}, "557d2cf7ba": {"quality": 0.01935483870967742, "cost": 0.009244542000000001, "time": 81.70965638160706}, "55c8aa8935": {"quality": 0.07660522273425499, "cost": 0.012967038, "time": 127.87031970024108}, "55e6bf8f14": {"quality": 0.10623655913978494, "cost": 0.031587628000000006, "time": 88.87307133674622}, "56a0660622": {"quality": 0.07333333333333333, "cost": 0.02785145, "time": 61.1418399810791}, "56a29a28c5": {"quality": 0.04, "cost": 0.005067498, "time": 67.83964855670928}, "56c4fd5056": {"quality": 0.0, "cost": 0.05983196800000001, "time": 88.72949783802034}, "5703697dbd": {"quality": 0.09268817204301075, "cost": 0.006353454, "time": 94.9607982635498}, "5718f2ed80": {"quality": 0.10946236559139785, "cost": 0.003271383, "time": 89.07395238876343}, "572a02a59a": {"quality": 0.15623655913978496, "cost": 0.004805790000000001, "time": 95.1497330904007}, "5750713a41": {"quality": 0.10301075268817204, "cost": 0.006427176, "time": 93.20527319908142}, "57757ef15e": {"quality": 0.06301075268817204, "cost": 0.006945941999999999, "time": 92.32843782901764}, "579c81bbe0": {"quality": 0.0, "cost": 0.005599266, "time": 66.2362874507904}, "57bed1722f": {"quality": 0.0, "cost": 0.005544702, "time": 79.84823746681214}, "585ba6d20b": {"quality": 0.10301075268817204, "cost": 0.007482849, "time": 135.71461656093598}, "589267ac64": {"quality": 0.043010752688172046, "cost": 0.0073127800000000005, "time": 65.76824603080749}, "589a1cea79": {"quality": 0.03612903225806452, "cost": 0.009729424, "time": 144.28782908916475}, "58ca42839b": {"quality": 0.10301075268817204, "cost": 0.026166294, "time": 107.9141107559204}, "58dc373441": {"quality": 0.10623655913978494, "cost": 0.031231837999999998, "time": 134.37797992229463}, "59006532b4": {"quality": 0.10301075268817204, "cost": 0.007373505, "time": 106.22529966831206}, "59326c4e00": {"quality": 0.0, "cost": 0.006372392, "time": 76.82031381130219}, "593975c75b": {"quality": 0.11268817204301075, "cost": 0.04324769700000001, "time": 137.53491501808168}, "596b4f8694": {"quality": 0.10301075268817204, "cost": 0.034689392, "time": 136.43740742206575}, "5971ba4e0d": {"quality": 0.10946236559139785, "cost": 0.00188677, "time": 45.389907240867615}, "5996465c0a": {"quality": 0.10623655913978494, "cost": 0.031329037000000004, "time": 141.25967452526095}, "59d70b9f65": {"quality": 0.10301075268817204, "cost": 0.03321219, "time": 110.12462825775147}, "59f887b67c": {"quality": 0.04, "cost": 0.010642958000000001, "time": 167.4949809551239}, "5a22920db4": {"quality": 0.10623655913978494, "cost": 0.0019284089999999999, "time": 76.70821943283082}, "5a35020d45": {"quality": 0.10623655913978494, "cost": 0.006820569, "time": 120.13344073295593}, "5aa43da1fc": {"quality": 0.11591397849462365, "cost": 0.028597824, "time": 106.1217529296875}, "5ae0d88127": {"quality": 0.0, "cost": 0.032261024000000006, "time": 121.66821670532227}, "5b10fbdbe1": {"quality": 0.11913978494623656, "cost": 0.014016424, "time": 136.450377035141}, "5bade9eb85": {"quality": 0.10623655913978494, "cost": 0.009162800999999998, "time": 132.39995594024657}, "5be16744bf": {"quality": 0.09729646697388633, "cost": 0.01510615, "time": 108.84415483474731}, "5c5055e252": {"quality": 0.0696774193548387, "cost": 0.029808262000000006, "time": 102.84497191905976}, "5c53feccd9": {"quality": 0.09333333333333334, "cost": 0.016134796, "time": 133.5285586118698}, "5c77c7c2b2": {"quality": 0.08301075268817204, "cost": 0.009062296000000001, "time": 106.83322401046752}, "5d298b5b48": {"quality": 0.10623655913978494, "cost": 0.037192654000000006, "time": 126.51887106895447}, "5d41515d2e": {"quality": 0.10946236559139785, "cost": 0.027884363000000002, "time": 88.88220546245574}, "5d4babc723": {"quality": 0.12623655913978493, "cost": 0.033099258000000006, "time": 115.31343786716462}, "5d79b50feb": {"quality": 0.02258064516129032, "cost": 0.013859904000000003, "time": 97.84079582691191}, "5dc216cd6b": {"quality": 0.10946236559139785, "cost": 0.010709733999999999, "time": 78.82858054637909}, "5dd68c1b8f": {"quality": 0.10301075268817204, "cost": 0.0015060479999999998, "time": 49.43739776611328}, "5de4a882c1": {"quality": 0.10623655913978494, "cost": 0.008982037, "time": 78.31139621734619}, "5e04e1c72d": {"quality": 0.10623655913978494, "cost": 0.040715788, "time": 81.60498032569885}, "5e923cee9e": {"quality": 0.10301075268817204, "cost": 0.017039563, "time": 113.84931592941284}, "5ea2fab380": {"quality": 0.08301075268817204, "cost": 0.004995846, "time": 52.83791854381562}, "5eb3bb525b": {"quality": 0.05333333333333333, "cost": 0.009461896000000001, "time": 96.70504157543182}, "5eea899380": {"quality": 0.11591397849462365, "cost": 0.037967743000000005, "time": 113.0098082780838}, "5f37b3902b": {"quality": 0.06258064516129032, "cost": 0.007317896000000001, "time": 72.8305846452713}, "5f9282df3c": {"quality": 0.13946236559139785, "cost": 0.018534208000000003, "time": 113.41562910079956}, "6019884cf3": {"quality": 0.10301075268817204, "cost": 0.035076015, "time": 124.4206553697586}, "606352363e": {"quality": 0.06301075268817204, "cost": 0.036100772, "time": 120.98173689842224}, "608728f868": {"quality": 0.08946236559139785, "cost": 0.033319759000000004, "time": 122.43799624443054}, "60b9e936f1": {"quality": 0.02, "cost": 0.027687884000000003, "time": 62.59779427051544}, "60cb623c53": {"quality": 0.0, "cost": 0.002459736, "time": 58.33065054416656}, "6234de86b4": {"quality": 0.0696774193548387, "cost": 0.04136126700000001, "time": 96.66275715827942}, "62352c6854": {"quality": 0.11591397849462365, "cost": 0.03903031800000001, "time": 117.9777235031128}, "63a0aaebed": {"quality": 0.03333333333333333, "cost": 0.002063613, "time": 61.364303207397455}, "647dda686f": {"quality": 0.11268817204301075, "cost": 0.016785803000000002, "time": 112.19475474357606}, "6511b21ded": {"quality": 0.10301075268817204, "cost": 0.013076242000000002, "time": 83.33274998664857}, "652c0f4bdf": {"quality": 0.02, "cost": 0.013811997999999999, "time": 122.77105476856232}, "6533c85913": {"quality": 0.056129032258064517, "cost": 0.010490979000000001, "time": 115.75537250041961}, "65627426e0": {"quality": 0.07333333333333333, "cost": 0.000720418, "time": 74.83012104034424}, "65b76da9c6": {"quality": 0.0696774193548387, "cost": 0.007407492, "time": 86.19350485801698}, "65be1c1306": {"quality": 0.08623655913978495, "cost": 0.001827702, "time": 35.30676467418671}, "65e0216208": {"quality": 0.0, "cost": 0.008113563, "time": 120.1523479938507}, "65eee615d7": {"quality": 0.12301075268817205, "cost": 0.00056168, "time": 33.19386031627655}, "6623d7a5ac": {"quality": 0.11591397849462365, "cost": 0.035060653000000004, "time": 117.17992658615111}, "66750c0934": {"quality": 0.10301075268817204, "cost": 0.007793503, "time": 92.1173137664795}, "66776ec181": {"quality": 0.09612903225806452, "cost": 0.007265367, "time": 135.96958916187288}, "66e5ae0a21": {"quality": 0.0, "cost": 0.021849754, "time": 69.76327166557311}, "6750a8d7a7": {"quality": 0.11591397849462365, "cost": 0.014042485, "time": 90.68010911941528}, "67632141f6": {"quality": 0.008333333333333333, "cost": 0.0051496020000000005, "time": 66.89973094463349}, "67868fcff6": {"quality": 0.11591397849462365, "cost": 0.01194172, "time": 93.14265236854553}, "67bab6732d": {"quality": 0.07612903225806453, "cost": 0.011601776, "time": 102.1346355676651}, "67fe399cf1": {"quality": 0.11913978494623656, "cost": 0.007208916, "time": 103.40100402832032}, "6846bd8fb3": {"quality": 0.09333333333333334, "cost": 0.007100624999999999, "time": 104.9791820526123}, "68b4cc3e39": {"quality": 0.12301075268817205, "cost": 0.045394822, "time": 143.03540830612184}, "69a029ae36": {"quality": 0.0, "cost": 0.028152226000000002, "time": 119.96623668670654}, "69b3b67de6": {"quality": 0.11591397849462365, "cost": 0.008464616999999999, "time": 114.51782112121583}, "69bf3f6ba0": {"quality": 0.08946236559139785, "cost": 0.009020988, "time": 113.73228361606598}, "6a022c3f73": {"quality": 0.06946236559139785, "cost": 0.004192794, "time": 110.7649359703064}, "6a10c53ad8": {"quality": 0.12697388632872503, "cost": 0.012509873000000001, "time": 133.7932172060013}, "6a6348f69d": {"quality": 0.04, "cost": 0.001041536, "time": 82.5766867876053}, "6a74a11bee": {"quality": 0.12623655913978493, "cost": 0.030354143, "time": 111.96903376579286}, "6a90ec29dd": {"quality": 0.11591397849462365, "cost": 0.01421163, "time": 101.9924022436142}, "6aac59742a": {"quality": 0.00967741935483871, "cost": 0.009429429, "time": 145.55764169692992}, "6b0862c597": {"quality": 0.10623655913978494, "cost": 0.028999667000000007, "time": 90.02094576358795}, "6b3c16def2": {"quality": 0.10697388632872504, "cost": 0.001518831, "time": 61.66853258609771}, "6b99b0e901": {"quality": 0.20301075268817204, "cost": 0.01664123, "time": 88.09062700271608}, "6b9b2b3515": {"quality": 0.07333333333333333, "cost": 0.039732264, "time": 98.20316002368926}, "6bcc02962b": {"quality": 0.10946236559139785, "cost": 0.031133424, "time": 97.9912957429886}, "6c1987a9e3": {"quality": 0.12052227342549923, "cost": 0.034681694000000006, "time": 130.62333233356475}, "6c50123ee1": {"quality": 0.10301075268817204, "cost": 0.027581610000000003, "time": 51.59311645030975}, "6c67c36480": {"quality": 0.02, "cost": 0.034065069, "time": 137.40613181591033}, "6c9b9f1363": {"quality": 0.10623655913978494, "cost": 0.033907465, "time": 132.98066565990447}, "6cc813aa68": {"quality": 0.04258064516129032, "cost": 0.006083242000000001, "time": 64.82899880409241}, "6d20c6ace0": {"quality": 0.12301075268817205, "cost": 0.026338517999999998, "time": 129.11275482177734}, "6d444fe21a": {"quality": 0.08623655913978495, "cost": 0.011278744, "time": 98.86997501850128}, "6db70dc3b6": {"quality": 0.07333333333333333, "cost": 0.001547284, "time": 91.66993854045867}, "6e0690f576": {"quality": 0.02, "cost": 0.001789902, "time": 73.95127286911011}, "6e06cc804f": {"quality": 0.10301075268817204, "cost": 0.06912567600000001, "time": 111.87644176483155}, "6e24048a2e": {"quality": 0.10623655913978494, "cost": 0.020128832, "time": 86.35281929969787}, "6e93514f45": {"quality": 0.0, "cost": 0.005360688000000001, "time": 106.13040091991425}, "6eae47102b": {"quality": 0.10301075268817204, "cost": 0.008400768, "time": 124.15161423683166}, "6ed4cae469": {"quality": 0.10301075268817204, "cost": 0.007222452, "time": 52.79307363033295}, "6ef3b7127e": {"quality": 0.20946236559139786, "cost": 0.006944608000000001, "time": 77.88400571346283}, "6f60a05c33": {"quality": 0.06623655913978495, "cost": 0.0043084709999999995, "time": 119.35040576457976}, "6fe0b3f929": {"quality": 0.0, "cost": 0.006453737999999999, "time": 100.1590167760849}, "700ab1d309": {"quality": 0.0, "cost": 0.017191588, "time": 130.72283561229705}, "7040e83d52": {"quality": 0.07333333333333333, "cost": 0.021663679, "time": 123.93634688854218}, "7046765af8": {"quality": 0.10301075268817204, "cost": 0.035322934, "time": 155.01265740394592}, "70b7c92ce8": {"quality": 0.11456989247311827, "cost": 0.030668973000000002, "time": 145.30480256080628}, "70c850e039": {"quality": 0.0696774193548387, "cost": 0.0025364249999999997, "time": 93.23688249588012}, "7112a7e64c": {"quality": 0.10946236559139785, "cost": 0.005931566000000001, "time": 86.47371196746826}, "71b615468b": {"quality": 0.06, "cost": 0.011086878000000001, "time": 121.78951773643495}, "723fd5589a": {"quality": 0.0, "cost": 0.013953277000000002, "time": 128.76966230869294}, "7250da0f41": {"quality": 0.0, "cost": 0.007366451999999999, "time": 91.1376808166504}, "7274a50778": {"quality": 0.10301075268817204, "cost": 0.033229607999999994, "time": 102.63909165859224}, "72d022ce33": {"quality": 0.10301075268817204, "cost": 0.04128840500000001, "time": 126.40671186447145}, "7347cf0308": {"quality": 0.0, "cost": 0.0036175589999999994, "time": 66.07082681655885}, "736e652158": {"quality": 0.0, "cost": 0.006011901, "time": 124.95026433467865}, "739b1f81dc": {"quality": 0.10301075268817204, "cost": 0.001929351, "time": 59.82662603855133}, "742a5c0552": {"quality": 0.10623655913978494, "cost": 0.028237606000000005, "time": 58.23579893112183}, "742ec1b2e1": {"quality": 0.049677419354838714, "cost": 0.029807655000000002, "time": 101.90871527194977}, "7435fd54f8": {"quality": 0.06946236559139785, "cost": 0.030427277000000003, "time": 121.13120923042297}, "7445d99939": {"quality": 0.00967741935483871, "cost": 0.0071708580000000004, "time": 93.88335957527161}, "7466a5f424": {"quality": 0.21591397849462365, "cost": 0.015802706, "time": 113.56059277057648}, "74a0be215b": {"quality": 0.10623655913978494, "cost": 0.033583158, "time": 125.14361772537231}, "74d7f64b8c": {"quality": 0.0, "cost": 0.006800163, "time": 146.70624086856844}, "7524905580": {"quality": 0.10623655913978494, "cost": 0.004893289, "time": 87.59031102657318}, "752d9649f2": {"quality": 0.11268817204301075, "cost": 0.013868582, "time": 121.90489645004273}, "75d61c2cd0": {"quality": 0.10301075268817204, "cost": 0.029877515, "time": 85.95328326225281}, "7604c0aa13": {"quality": 0.10301075268817204, "cost": 0.010774968000000001, "time": 84.55494666099548}, "76c09db721": {"quality": 0.10301075268817204, "cost": 0.016476853, "time": 132.14744505882265}, "774f268b66": {"quality": 0.10946236559139785, "cost": 0.038069169, "time": 123.8245237827301}, "7765576286": {"quality": 0.09333333333333334, "cost": 0.008933451, "time": 130.5573842048645}, "77c02b00c1": {"quality": 0.05655913978494624, "cost": 0.011193958, "time": 124.06667771339417}, "7801da66b9": {"quality": 0.0, "cost": 0.004612473000000001, "time": 94.23449354171753}, "782d52674e": {"quality": 0.10301075268817204, "cost": 0.034843558000000004, "time": 141.27741813659668}, "786e5d0af5": {"quality": 0.10946236559139785, "cost": 0.009984510999999998, "time": 130.25511062145233}, "7878563d63": {"quality": 0.10946236559139785, "cost": 0.009147046, "time": 73.60474834442138}, "79e1ca9b3c": {"quality": 0.0, "cost": 0.02973492, "time": 82.63413181304932}, "79fad58f07": {"quality": 0.12623655913978493, "cost": 0.002492818, "time": 58.73518245220184}, "7a207b42a8": {"quality": 0.0, "cost": 0.008606742, "time": 142.3234792470932}, "7a2cdc546c": {"quality": 0.10301075268817204, "cost": 0.032290785, "time": 102.24121663570403}, "7a42a77788": {"quality": 0.10047619047619048, "cost": 0.031140232, "time": 94.29256238937378}, "7a58d3472b": {"quality": 0.09333333333333334, "cost": 0.013635729000000001, "time": 102.35271875858307}, "7b3937c1f1": {"quality": 0.10301075268817204, "cost": 0.009220510000000001, "time": 54.74555864334106}, "7b6f44618e": {"quality": 0.06946236559139785, "cost": 0.013006586, "time": 130.8397382736206}, "7c62576527": {"quality": 0.22424731182795699, "cost": 0.023333732000000003, "time": 129.530002951622}, "7c89a2b69e": {"quality": 0.0, "cost": 0.012572941, "time": 131.1510176181793}, "7c96c9712f": {"quality": 0.12301075268817205, "cost": 0.02800367, "time": 71.56067564487458}, "7ca066aa1c": {"quality": 0.10301075268817204, "cost": 0.017135469, "time": 126.58104584217071}, "7cb5591f27": {"quality": 0.12301075268817205, "cost": 0.037874200000000004, "time": 99.9374593257904}, "7cf56a7fbc": {"quality": 0.0, "cost": 0.03754542600000001, "time": 81.54608149528502}, "7d44f0959d": {"quality": 0.11015360983102919, "cost": 0.006577314, "time": 95.78297588825225}, "7d60c38c5c": {"quality": 0.10301075268817204, "cost": 0.0019278, "time": 62.340004205703735}, "7d9b4535ac": {"quality": 0.12301075268817205, "cost": 0.030719909000000004, "time": 124.86560246944427}, "7daf7ff182": {"quality": 0.21591397849462365, "cost": 0.011567735999999999, "time": 86.85500297546386}, "7dcedb3d02": {"quality": 0.11591397849462365, "cost": 0.034740937, "time": 118.09863307476044}, "7e22f12cd1": {"quality": 0.10946236559139785, "cost": 0.012891924999999999, "time": 121.22575757503509}, "7e53a50b13": {"quality": 0.10946236559139785, "cost": 0.014215881, "time": 90.20128819942474}, "7ed07ad40a": {"quality": 0.10623655913978494, "cost": 0.006843809999999999, "time": 95.13763875961303}, "7fa67a7656": {"quality": 0.06623655913978495, "cost": 0.008441146000000002, "time": 80.71841621398926}, "7fc6c84bdf": {"quality": 0.10301075268817204, "cost": 0.014648633, "time": 94.4585831642151}, "7ff8a779cc": {"quality": 0.049677419354838714, "cost": 0.033345343, "time": 90.90201032161713}, "801af99400": {"quality": 0.12623655913978493, "cost": 0.031287835, "time": 92.26307544708251}, "806881adcb": {"quality": 0.10623655913978494, "cost": 0.009180413, "time": 103.7968935251236}, "80ad4122e4": {"quality": 0.10946236559139785, "cost": 0.033705702000000004, "time": 147.08289949893953}, "80be7df955": {"quality": 0.0, "cost": 0.001751958, "time": 72.68495740890503}, "80bf60c422": {"quality": 0.0, "cost": 0.002103153, "time": 69.09833533763886}, "81333c7a33": {"quality": 0.19333333333333333, "cost": 0.018656088, "time": 122.73940353393554}, "813e75210b": {"quality": 0.12623655913978493, "cost": 0.002453274, "time": 31.607514262199402}, "81660ae8b2": {"quality": 0.00967741935483871, "cost": 0.03510192400000001, "time": 159.472354388237}, "816958b5d1": {"quality": 0.10301075268817204, "cost": 0.033492955000000005, "time": 150.83300273418428}, "81ab2ef3f4": {"quality": 0.08301075268817204, "cost": 0.015009652000000002, "time": 122.83408730030061}, "829df73946": {"quality": 0.10623655913978494, "cost": 0.0027339649999999997, "time": 95.04270524978638}, "82ea1bd1b9": {"quality": 0.10301075268817204, "cost": 0.036581163, "time": 130.3844569683075}, "8357183895": {"quality": 0.0, "cost": 0.009140319000000001, "time": 168.7040484428406}, "8392a6083a": {"quality": 0.11591397849462365, "cost": 0.013724715999999998, "time": 170.04993345737458}, "83aee532b7": {"quality": 0.0, "cost": 0.032044618999999996, "time": 169.97980024814606}, "83b26646c3": {"quality": 0.10946236559139785, "cost": 0.027026598000000002, "time": 126.73678522109986}, "83c9e66ec6": {"quality": 0.10623655913978494, "cost": 0.0077078630000000006, "time": 122.85937674045563}, "847a0e5db5": {"quality": 0.10301075268817204, "cost": 0.033220362, "time": 118.9956719636917}, "847fd49235": {"quality": 0.10623655913978494, "cost": 0.004587912, "time": 120.00769526958466}, "849100224d": {"quality": 0.11729646697388633, "cost": 0.040131096000000005, "time": 162.80983967781066}, "84b91c37ab": {"quality": 0.10301075268817204, "cost": 0.035888904, "time": 185.50609340667725}, "8519bef585": {"quality": 0.0, "cost": 0.002883114, "time": 83.83753573894501}, "85c94a5505": {"quality": 0.10623655913978494, "cost": 0.003060162, "time": 47.44887166023254}, "85eda38404": {"quality": 0.06301075268817204, "cost": 0.00743115, "time": 97.06213176250458}, "862183bfb9": {"quality": 0.11591397849462365, "cost": 0.007656215, "time": 150.60711097717285}, "8631e49c94": {"quality": 0.10623655913978494, "cost": 0.032397725, "time": 178.85168232917786}, "8668f65f05": {"quality": 0.10301075268817204, "cost": 0.010037819, "time": 170.7291125535965}, "86bf6375af": {"quality": 0.07290322580645162, "cost": 0.034898615, "time": 167.93282201290128}, "870e2f87b4": {"quality": 0.09333333333333334, "cost": 0.014351701000000001, "time": 174.38446729183198}, "887ad124e1": {"quality": 0.10623655913978494, "cost": 0.0033872489999999997, "time": 141.54452052116392}, "8886cb3082": {"quality": 0.10623655913978494, "cost": 0.010027301, "time": 168.62686750888824}, "88e71efa9b": {"quality": 0.11591397849462365, "cost": 0.024657334, "time": 163.67923073768617}, "8941621423": {"quality": 0.10623655913978494, "cost": 0.006795555, "time": 135.67179551124573}, "8950a6efe0": {"quality": 0.10946236559139785, "cost": 0.029011140000000005, "time": 120.09507937431336}, "8961e4d901": {"quality": 0.02, "cost": 0.007544093999999999, "time": 115.46372551918029}, "8974aa89a0": {"quality": 0.08301075268817204, "cost": 0.0016073620000000002, "time": 48.72573773860931}, "89836d2020": {"quality": 0.21591397849462365, "cost": 0.03316775000000001, "time": 93.58950910568237}, "89a289907e": {"quality": 0.10301075268817204, "cost": 0.009082531000000001, "time": 127.31396248340606}, "89a35a09b1": {"quality": 0.10623655913978494, "cost": 0.007982379000000001, "time": 124.18780100345612}, "8a3a35c762": {"quality": 0.10946236559139785, "cost": 0.03612732600000001, "time": 119.14178504943848}, "8a50695d1f": {"quality": 0.043010752688172046, "cost": 0.016531714000000003, "time": 100.70311589241028}, "8ab351aa13": {"quality": 0.0, "cost": 0.040063444000000004, "time": 105.603106880188}, "8ac8b5773a": {"quality": 0.0696774193548387, "cost": 0.033945373, "time": 124.25757415294648}, "8acd758b7f": {"quality": 0.10623655913978494, "cost": 0.004647693, "time": 101.40544395446777}, "8b10891ea5": {"quality": 0.11333333333333334, "cost": 0.038892921999999996, "time": 124.35444567203521}, "8b721bbc6f": {"quality": 0.10946236559139785, "cost": 0.008056791, "time": 122.71747550964355}, "8b77535cce": {"quality": 0.12623655913978493, "cost": 0.035407286999999996, "time": 130.12753999233246}, "8bbbe0f52a": {"quality": 0.09279569892473119, "cost": 0.006340248, "time": 96.97207028865814}, "8bc184f385": {"quality": 0.10623655913978494, "cost": 0.0073677989999999995, "time": 97.76727423667907}, "8bf5c3eadc": {"quality": 0.02, "cost": 0.006312036, "time": 82.65424501895905}, "8bf80a50cb": {"quality": 0.11591397849462365, "cost": 0.0346818, "time": 110.19822652339934}, "8c274ca255": {"quality": 0.12301075268817205, "cost": 0.036505015, "time": 115.35032222270965}, "8d7594020b": {"quality": 0.12623655913978493, "cost": 0.029785446, "time": 79.58442108631134}, "8d79e03266": {"quality": 0.10301075268817204, "cost": 0.008962553, "time": 91.70296292304992}, "8d90814b94": {"quality": 0.21591397849462365, "cost": 0.009549217000000002, "time": 107.88517692089081}, "8e1a01da19": {"quality": 0.00967741935483871, "cost": 0.030469137000000007, "time": 110.28689606189728}, "8e2498635d": {"quality": 0.12946236559139784, "cost": 0.067304136, "time": 88.78618555068971}, "8e5842ccbd": {"quality": 0.12623655913978493, "cost": 0.0035327910000000004, "time": 61.36363306045533}, "8e5daf241e": {"quality": 0.10301075268817204, "cost": 0.029329670000000002, "time": 80.64679687023164}, "8e9715ee01": {"quality": 0.13268817204301075, "cost": 0.034420532000000004, "time": 73.95220372676849}, "8e9b7300d4": {"quality": 0.10301075268817204, "cost": 0.029812478000000003, "time": 81.88405811786652}, "8f29fab8ac": {"quality": 0.05333333333333333, "cost": 0.034482348, "time": 119.19576361179352}, "8f44d89429": {"quality": 0.20301075268817204, "cost": 0.034302998, "time": 86.89617729187012}, "8f4caddfe6": {"quality": 0.07935483870967741, "cost": 0.014937176, "time": 100.91416237354278}, "8f4edde3f0": {"quality": 0.10623655913978494, "cost": 0.017718742000000003, "time": 124.73474152088164}, "8f9cefbc22": {"quality": 0.10623655913978494, "cost": 0.028400606000000002, "time": 59.505778431892395}, "9025e2480f": {"quality": 0.11268817204301075, "cost": 0.006730606, "time": 74.51114990711213}, "9028588af4": {"quality": 0.10623655913978494, "cost": 0.00524569, "time": 45.83101971149445}, "9059fd80ad": {"quality": 0.00967741935483871, "cost": 0.0016962140000000001, "time": 79.82600400447845}, "90d5e40c1b": {"quality": 0.0, "cost": 0.006609534, "time": 97.84093914031982}, "90d9a86a2a": {"quality": 0.0, "cost": 0.0016498349999999997, "time": 50.78291637897492}, "90ff13783c": {"quality": 0.10301075268817204, "cost": 0.010615371000000002, "time": 137.4936174631119}, "90ff8eb055": {"quality": 0.029677419354838707, "cost": 0.010129776, "time": 135.72056851387023}, "9104e31369": {"quality": 0.09333333333333334, "cost": 0.013592545000000001, "time": 145.33199377059935}, "918983323f": {"quality": 0.00967741935483871, "cost": 0.0012531780000000002, "time": 41.7289758682251}, "91928dfdd9": {"quality": 0.10623655913978494, "cost": 0.029831955000000007, "time": 108.31667373180389}, "91c800af6b": {"quality": 0.08301075268817204, "cost": 0.011924733, "time": 149.60272026062012}, "91e841cfd5": {"quality": 0.11268817204301075, "cost": 0.035107227000000005, "time": 144.8135196208954}, "9253901a1f": {"quality": 0.0, "cost": 0.038044404000000004, "time": 144.24513857364656}, "9288642e53": {"quality": 0.02, "cost": 0.006087138000000001, "time": 72.39175655841828}, "92ba9c5be3": {"quality": 0.13591397849462367, "cost": 0.030314066, "time": 89.64683423042297}, "92c9dcd43b": {"quality": 0.10301075268817204, "cost": 0.009152825, "time": 133.89202611446382}, "93011c0821": {"quality": 0.03935483870967742, "cost": 0.012697265000000001, "time": 135.9506582260132}, "933b4d17dd": {"quality": 0.10301075268817204, "cost": 0.004906763, "time": 94.6856369972229}, "9373267bdb": {"quality": 0.12623655913978493, "cost": 0.03234980700000001, "time": 123.84810786247253}, "94010928c6": {"quality": 0.08623655913978495, "cost": 0.007078176, "time": 105.30463340282441}, "9403809e44": {"quality": 0.08301075268817204, "cost": 0.012096366, "time": 135.0477834701538}, "940c88ddc5": {"quality": 0.06946236559139785, "cost": 0.010277144, "time": 97.58399150371551}, "948f4081ba": {"quality": 0.10623655913978494, "cost": 0.015952197, "time": 132.39694006443023}, "94ac356663": {"quality": 0.10301075268817204, "cost": 0.007538082, "time": 100.63510990142822}, "94dff9a424": {"quality": 0.10301075268817204, "cost": 0.0058817290000000005, "time": 97.47921371459961}, "9508356a2e": {"quality": 0.0, "cost": 0.0074193060000000005, "time": 119.1678344964981}, "9539d0e28c": {"quality": 0.11913978494623656, "cost": 0.016965942000000005, "time": 127.95936200618743}, "956bdcc254": {"quality": 0.10301075268817204, "cost": 0.016353198000000003, "time": 113.47469205856324}, "957d0dafc5": {"quality": 0.12623655913978493, "cost": 0.03451419800000001, "time": 99.19511275291444}, "9594b0c783": {"quality": 0.06946236559139785, "cost": 0.0033617670000000003, "time": 103.66688406467438}, "95a7b80c2a": {"quality": 0.10301075268817204, "cost": 0.012909755, "time": 96.70905361175537}, "964c671f18": {"quality": 0.21591397849462365, "cost": 0.012280253000000001, "time": 130.05689902305602}, "9679fe2b69": {"quality": 0.10301075268817204, "cost": 0.003347709, "time": 92.87826173305511}, "968fc95038": {"quality": 0.10301075268817204, "cost": 0.0036856379999999993, "time": 98.10628998279572}, "96b487c724": {"quality": 0.08946236559139785, "cost": 0.00053203, "time": 54.895061421394345}, "96c30205f5": {"quality": 0.10623655913978494, "cost": 0.031391626, "time": 97.40448112487793}, "96f87d6483": {"quality": 0.07333333333333333, "cost": 0.012635357, "time": 153.3388079404831}, "972c83b002": {"quality": 0.029677419354838707, "cost": 0.007475796, "time": 118.32212336063385}, "977a4d6b6b": {"quality": 0.13591397849462367, "cost": 0.01375324, "time": 138.46970500946043}, "97bc30bd83": {"quality": 0.07612903225806453, "cost": 0.010801506999999998, "time": 111.81023037433624}, "980db5f95f": {"quality": 0.10301075268817204, "cost": 0.03036672, "time": 113.69273748397828}, "9836765d41": {"quality": 0.12301075268817205, "cost": 0.032121622, "time": 104.68945379257201}, "99569e3937": {"quality": 0.10623655913978494, "cost": 0.035286567000000005, "time": 141.43686013221742}, "99ea16a9a6": {"quality": 0.10623655913978494, "cost": 0.009821013, "time": 108.473588347435}, "9a5b39370f": {"quality": 0.0, "cost": 0.036045704, "time": 155.65953776836395}, "9aa4abfb50": {"quality": 0.0, "cost": 0.003652884, "time": 89.60838494300842}, "9ad7a98c31": {"quality": 0.12301075268817205, "cost": 0.03251525400000001, "time": 111.1855703830719}, "9b3fb79bcb": {"quality": 0.12301075268817205, "cost": 0.010473083, "time": 107.05149478912352}, "9b6d4915f3": {"quality": 0.0, "cost": 0.00475386, "time": 53.93124535083771}, "9bae5bafc1": {"quality": 0.09333333333333334, "cost": 0.013743354000000001, "time": 105.66356384754181}, "9be8a5f317": {"quality": 0.10623655913978494, "cost": 0.038140224, "time": 141.94463317394258}, "9c549db0a7": {"quality": 0.0, "cost": 0.011336479, "time": 83.88670737743377}, "9c85f8cfcb": {"quality": 0.0, "cost": 0.002950311, "time": 71.82748806476593}, "9c8cc46e6c": {"quality": 0.06623655913978495, "cost": 0.0019796609999999997, "time": 72.90286383628845}, "9c97d35a30": {"quality": 0.10301075268817204, "cost": 0.005516262, "time": 115.19557769298552}, "9cbe7858a2": {"quality": 0.11333333333333334, "cost": 0.034743496000000006, "time": 140.37097189426424}, "9ce2c3fd98": {"quality": 0.0, "cost": 0.004397076, "time": 118.44574811458588}, "9d18cd0737": {"quality": 0.06301075268817204, "cost": 0.004985955, "time": 77.21244201660156}, "9e06360bc9": {"quality": 0.10946236559139785, "cost": 0.008386245, "time": 112.56666424274445}, "9f07a95e69": {"quality": 0.12301075268817205, "cost": 0.039345112, "time": 142.53303418159484}, "9fb157be35": {"quality": 0.04, "cost": 0.009244232, "time": 117.06155586242676}, "a04ac8e33a": {"quality": 0.0, "cost": 0.030044402000000005, "time": 83.63084626197815}, "a04bc6e116": {"quality": 0.04967741935483871, "cost": 0.029843946, "time": 100.13187789916992}, "a0b81be5b4": {"quality": 0.10301075268817204, "cost": 0.0022555170000000003, "time": 77.33556809425355}, "a0c85d260e": {"quality": 0.10623655913978494, "cost": 0.013999365, "time": 147.74334559440612}, "a0dc9f50ac": {"quality": 0.012903225806451613, "cost": 0.006003528, "time": 77.70861837863922}, "a18225b7b5": {"quality": 0.10301075268817204, "cost": 0.033204910000000004, "time": 99.84152095317842}, "a1d822289e": {"quality": 0.10301075268817204, "cost": 0.03703391, "time": 105.16314966678618}, "a25596c056": {"quality": 0.06301075268817204, "cost": 0.034485563999999996, "time": 138.92013294696807}, "a2811c7324": {"quality": 0.10623655913978494, "cost": 0.009388681000000001, "time": 112.0317987203598}, "a2aa082d14": {"quality": 0.0, "cost": 0.014266204, "time": 101.78627176284789}, "a2cd339ad9": {"quality": 0.09333333333333334, "cost": 0.010370297, "time": 147.42609903812408}, "a2fd03e6a5": {"quality": 0.07655913978494623, "cost": 0.004336301999999999, "time": 36.67434537410736}, "a31e87d7cb": {"quality": 0.06301075268817204, "cost": 0.001581552, "time": 76.9124368429184}, "a3e23c327b": {"quality": 0.06946236559139785, "cost": 0.03564489000000001, "time": 150.25721719264985}, "a457f6c300": {"quality": 0.0, "cost": 0.007949861999999999, "time": 132.3079260349274}, "a47de025c8": {"quality": 0.0, "cost": 0.004268559, "time": 89.55218677520752}, "a515a9c8cc": {"quality": 0.05333333333333333, "cost": 0.002060598, "time": 76.20932059288026}, "a5949b76ec": {"quality": 0.0, "cost": 0.002426688, "time": 69.78927192687988}, "a60dd076b8": {"quality": 0.10623655913978494, "cost": 0.002494836, "time": 77.4125370502472}, "a6297a6c56": {"quality": 0.06623655913978495, "cost": 0.0024065099999999997, "time": 65.99698326587676}, "a62b7555b9": {"quality": 0.08258064516129032, "cost": 0.03560092200000001, "time": 131.32008333206176}, "a63f48e8ca": {"quality": 0.0, "cost": 0.029433226000000003, "time": 74.86844005584717}, "a6460dbb7c": {"quality": 0.10623655913978494, "cost": 0.002761025, "time": 64.81612529754639}, "a66a4cf4b0": {"quality": 0.11591397849462365, "cost": 0.03962680600000001, "time": 131.48869993686674}, "a6e2d69222": {"quality": 0.10301075268817204, "cost": 0.029019002, "time": 97.08205032348633}, "a717c4c535": {"quality": 0.05290322580645161, "cost": 0.013103212, "time": 130.28173251152037}, "a76afe9960": {"quality": 0.12301075268817205, "cost": 0.030138277, "time": 103.21392459869384}, "a7a6353090": {"quality": 0.10623655913978494, "cost": 0.031531417000000006, "time": 103.30408613681793}, "a80f6535b1": {"quality": 0.06301075268817204, "cost": 0.005650186, "time": 66.3761245250702}, "a86b137d7f": {"quality": 0.06946236559139785, "cost": 0.002503022, "time": 67.13813652992249}, "a88eb1493c": {"quality": 0.10301075268817204, "cost": 0.0049795349999999985, "time": 64.7448471069336}, "a89c533d6c": {"quality": 0.0, "cost": 0.027630250000000002, "time": 46.682720041275026}, "a8d8264600": {"quality": 0.1296774193548387, "cost": 0.035944667, "time": 153.82529056072235}, "a8eb36b210": {"quality": 0.10301075268817204, "cost": 0.023975060000000003, "time": 108.62396328449249}, "a95b4a6dd0": {"quality": 0.08301075268817204, "cost": 0.00891948, "time": 111.40819778442383}, "a9621ea4e6": {"quality": 0.043010752688172046, "cost": 0.00758955, "time": 111.94071905612945}, "a9721a0a50": {"quality": 0.11268817204301075, "cost": 0.006963254, "time": 83.20404796600342}, "a972b02c61": {"quality": 0.12301075268817205, "cost": 0.016982312, "time": 36.22950010299682}, "a9c5c4e311": {"quality": 0.11456989247311827, "cost": 0.009789541, "time": 112.16675176620484}, "a9d96670eb": {"quality": 0.12623655913978493, "cost": 0.028751066000000002, "time": 111.07726712226868}, "a9e8c974d3": {"quality": 0.0, "cost": 0.03628516, "time": 154.63389499187468}, "aa08180e36": {"quality": 0.07591397849462365, "cost": 0.010442482000000001, "time": 149.7024934768677}, "aa38702a02": {"quality": 0.06623655913978495, "cost": 0.00453825, "time": 115.99718639850616}, "aadbfc418b": {"quality": 0.15301075268817205, "cost": 0.0039624659999999996, "time": 115.4608172416687}, "aaeb8b0010": {"quality": 0.03612903225806452, "cost": 0.033459904, "time": 146.18061661720276}, "ab288ee7f2": {"quality": 0.043010752688172046, "cost": 0.031835803, "time": 151.1489573955536}, "ab43b02cb0": {"quality": 0.0, "cost": 0.010940676, "time": 85.55018713474274}, "aba1d612cc": {"quality": 0.10301075268817204, "cost": 0.04242697200000001, "time": 129.53015527725222}, "ac208e7a1d": {"quality": 0.20623655913978495, "cost": 0.010061258, "time": 87.22576036453248}, "ac2224adbe": {"quality": 0.0, "cost": 0.026542246000000005, "time": 73.37120480537413}, "ac828ffe70": {"quality": 0.07655913978494625, "cost": 0.000756074, "time": 55.469612050056455}, "ac9fdc1550": {"quality": 0.10301075268817204, "cost": 0.013684735, "time": 102.56252949237823}, "aca957ecff": {"quality": 0.11456989247311827, "cost": 0.03768476100000001, "time": 152.26708533763883}, "acfe1ed920": {"quality": 0.09456989247311828, "cost": 0.005188744, "time": 57.178893375396726}, "ad3efe44c3": {"quality": 0.0, "cost": 0.002697534, "time": 72.25062220096589}, "ad41c95a99": {"quality": 0.10623655913978494, "cost": 0.025689668000000002, "time": 115.99265196323395}, "ad48432c22": {"quality": 0.0, "cost": 0.009775953, "time": 191.9520122528076}, "ad6ebbba8d": {"quality": 0.0, "cost": 0.007917252999999999, "time": 140.29350352287292}, "ad90055ef6": {"quality": 0.0, "cost": 0.003889926, "time": 112.45582339763641}, "adab1e0fb1": {"quality": 0.10301075268817204, "cost": 0.006015302, "time": 91.00434277057647}, "ae655ec593": {"quality": 0.08301075268817204, "cost": 0.004026768, "time": 141.7740585565567}, "ae94b172be": {"quality": 0.06301075268817204, "cost": 0.007697588, "time": 143.52500240802766}, "aec9dc5873": {"quality": 0.0, "cost": 0.027518260000000003, "time": 96.50754041671753}, "af360c323c": {"quality": 0.0, "cost": 0.0033090330000000003, "time": 149.9352635383606}, "af90567194": {"quality": 0.10623655913978494, "cost": 0.013733104999999999, "time": 181.40714287757874}, "afe77d0f89": {"quality": 0.11591397849462365, "cost": 0.03486629, "time": 177.79787800312042}, "b0948c05b6": {"quality": 0.09333333333333334, "cost": 0.01202388, "time": 110.92746062278746}, "b0c4a6640b": {"quality": 0.10301075268817204, "cost": 0.031655476, "time": 155.44975624084475}, "b12caafd58": {"quality": 0.06301075268817204, "cost": 0.03197457, "time": 155.3449794769287}, "b18168b9c1": {"quality": 0.08301075268817204, "cost": 0.0077949479999999995, "time": 119.46661601066589}, "b1a7428a01": {"quality": 0.12301075268817205, "cost": 0.034865056000000005, "time": 145.9925803422928}, "b1acdebb48": {"quality": 0.0929032258064516, "cost": 0.04751790600000001, "time": 124.27810626029968}, "b1b06f4ee7": {"quality": 0.0, "cost": 0.05763750000000001, "time": 95.25730528831483}, "b1cf8d33e5": {"quality": 0.08301075268817204, "cost": 0.007401675000000001, "time": 103.68885869979857}, "b1e9ab6b1a": {"quality": 0.10946236559139785, "cost": 0.038297786, "time": 146.62364199161527}, "b2b057ba41": {"quality": 0.10301075268817204, "cost": 0.00373947, "time": 102.88082673549653}, "b2e063499d": {"quality": 0.21591397849462365, "cost": 0.009003760999999999, "time": 103.52594914436341}, "b33412410e": {"quality": 0.0, "cost": 0.02474838, "time": 82.1103196144104}, "b3369775dc": {"quality": 0.10301075268817204, "cost": 0.012564327, "time": 147.59253644943237}, "b3b9205f60": {"quality": 0.07591397849462365, "cost": 0.020412146000000003, "time": 143.55707364082338}, "b3c56f0b3c": {"quality": 0.10623655913978494, "cost": 0.04188011999999999, "time": 104.512482714653}, "b3f20b706d": {"quality": 0.0, "cost": 0.0030317039999999996, "time": 98.11458349227905}, "b4002173ee": {"quality": 0.08301075268817204, "cost": 0.0054744, "time": 75.72368431091309}, "b46d382384": {"quality": 0.10301075268817204, "cost": 0.05967477, "time": 110.93994162082672}, "b4b2482ef9": {"quality": 0.07591397849462365, "cost": 0.033919934, "time": 114.34581046104432}, "b4be043238": {"quality": 0.12301075268817205, "cost": 0.04046709600000001, "time": 111.69091956615448}, "b531bd0548": {"quality": 0.10623655913978494, "cost": 0.042357856, "time": 138.67396595478056}, "b56c312eda": {"quality": 0.08301075268817204, "cost": 0.008939847, "time": 141.99621114730834}, "b5e2b41c1c": {"quality": 0.10623655913978494, "cost": 0.004302350999999999, "time": 110.59975845813752}, "b61ce57a90": {"quality": 0.11591397849462365, "cost": 0.020101276, "time": 140.674871301651}, "b64ddb14f9": {"quality": 0.10946236559139785, "cost": 0.0024013059999999998, "time": 41.2197571516037}, "b67107a43e": {"quality": 0.05333333333333333, "cost": 0.0056747790000000005, "time": 151.14496650695799}, "b67720aa5c": {"quality": 0.0, "cost": 0.029933376000000005, "time": 112.10192131996155}, "b682a23b89": {"quality": 0.10301075268817204, "cost": 0.004969453, "time": 109.97392725944519}, "b69ef5add4": {"quality": 0.04, "cost": 0.033153256000000006, "time": 127.61734595298768}, "b796b7ffd3": {"quality": 0.10946236559139785, "cost": 0.009056132000000001, "time": 156.37638986110687}, "b7a0083dc4": {"quality": 0.06, "cost": 0.029241969, "time": 111.82368907928466}, "b7d0e8557f": {"quality": 0.06623655913978495, "cost": 0.008024637, "time": 146.9499261379242}, "b8317a3a8c": {"quality": 0.10301075268817204, "cost": 0.040210949, "time": 126.69586458206176}, "b8ab3d2f25": {"quality": 0.11591397849462365, "cost": 0.008929705999999999, "time": 119.44116389751434}, "b8b569172f": {"quality": 0.10301075268817204, "cost": 0.010143565999999998, "time": 119.48254477977753}, "b8f5ab44bb": {"quality": 0.13591397849462367, "cost": 0.039590358000000006, "time": 144.63666880130768}, "b91e7fdb29": {"quality": 0.0, "cost": 0.009813995999999998, "time": 178.9033116340637}, "b932beaaa6": {"quality": 0.10301075268817204, "cost": 0.03316909800000001, "time": 101.8533546447754}, "b9770c2261": {"quality": 0.10301075268817204, "cost": 0.00194208, "time": 73.90803687572479}, "b9bb1e6f8d": {"quality": 0.06301075268817204, "cost": 0.012499610999999999, "time": 150.40043871402742}, "b9d0e8740c": {"quality": 0.10623655913978494, "cost": 0.034839880000000004, "time": 144.61329133510588}, "b9da208432": {"quality": 0.10301075268817204, "cost": 0.009705881, "time": 142.17393441200255}, "ba3223f6ac": {"quality": 0.04, "cost": 0.004067208, "time": 117.73131382465363}, "bb13365175": {"quality": 0.10946236559139785, "cost": 0.022399919, "time": 149.47980670928956}, "bb1b3a4d29": {"quality": 0.10301075268817204, "cost": 0.031956166, "time": 149.79417490959167}, "bb6536b0ab": {"quality": 0.06946236559139785, "cost": 0.010785974, "time": 102.99912581443786}, "bbba9dd6ae": {"quality": 0.11591397849462365, "cost": 0.005964354, "time": 77.64485285282134}, "bbde69a1ae": {"quality": 0.08967741935483872, "cost": 0.03738282200000001, "time": 164.79430203437806}, "bc29a0c0fe": {"quality": 0.11268817204301075, "cost": 0.033914098000000004, "time": 101.38525047302247}, "bc3d02f753": {"quality": 0.0, "cost": 0.0025910279999999996, "time": 93.81551551818848}, "bc4c1fcc64": {"quality": 0.20301075268817204, "cost": 0.0059535610000000004, "time": 67.91742668151855}, "bd30d27f62": {"quality": 0.12301075268817205, "cost": 0.062644566, "time": 121.11297037601472}, "bd99b2fb21": {"quality": 0.09333333333333334, "cost": 0.008204478, "time": 122.81714386940003}, "bddc7d2a34": {"quality": 0.10301075268817204, "cost": 0.03213537500000001, "time": 163.16414675712585}, "be2ae88f70": {"quality": 0.0, "cost": 0.004693302, "time": 137.78757257461547}, "be4740f38f": {"quality": 0.09655913978494624, "cost": 0.011966032000000001, "time": 123.94724879264831}, "bec0c6a95f": {"quality": 0.11134408602150538, "cost": 0.044889398, "time": 164.90446269512177}, "bed888d4dc": {"quality": 0.03612903225806452, "cost": 0.007983669, "time": 159.96076278686525}, "bf45e407f6": {"quality": 0.05290322580645161, "cost": 0.039748443999999994, "time": 127.84493451118469}, "bf5550f320": {"quality": 0.07333333333333333, "cost": 0.035326152, "time": 161.38907580375673}, "bf87e58322": {"quality": 0.10301075268817204, "cost": 0.02893854, "time": 130.29030165672305}, "bfed7670ed": {"quality": 0.08301075268817204, "cost": 0.010171252, "time": 128.07581346035005}, "c0541e2220": {"quality": 0.0, "cost": 0.037621554, "time": 99.32873375415802}, "c0e10c0048": {"quality": 0.0, "cost": 0.03382517, "time": 166.7884413957596}, "c127509a7a": {"quality": 0.12301075268817205, "cost": 0.013034976, "time": 122.95744087696076}, "c13682c7c7": {"quality": 0.11258064516129032, "cost": 0.012015295999999998, "time": 151.76094684600832}, "c13d6e78e9": {"quality": 0.12301075268817205, "cost": 0.017051258000000003, "time": 136.8413455247879}, "c14ff3144d": {"quality": 0.06623655913978495, "cost": 0.0015566399999999998, "time": 67.90232226848602}, "c1e42ac47b": {"quality": 0.10946236559139785, "cost": 0.039353246, "time": 155.5742819786072}, "c2949aa902": {"quality": 0.10301075268817204, "cost": 0.007100054999999999, "time": 123.65010952949524}, "c31e956b35": {"quality": 0.10301075268817204, "cost": 0.009654768000000001, "time": 160.06434454917905}, "c36b525dde": {"quality": 0.10623655913978494, "cost": 0.003221145, "time": 89.18764023780822}, "c38326e2bd": {"quality": 0.10946236559139785, "cost": 0.036419284, "time": 149.7904277563095}, "c3ec2cec59": {"quality": 0.11591397849462365, "cost": 0.012373450000000001, "time": 124.03460397720337}, "c44720575f": {"quality": 0.10623655913978494, "cost": 0.011866406, "time": 114.06694071292877}, "c48ecefab6": {"quality": 0.0, "cost": 0.031072968000000006, "time": 158.07063992023467}, "c4a64eb40f": {"quality": 0.10301075268817204, "cost": 0.038885108, "time": 112.12137053012847}, "c4a80d19b3": {"quality": 0.10301075268817204, "cost": 0.01095102, "time": 163.5285721540451}, "c4c2826afd": {"quality": 0.02258064516129032, "cost": 0.009101021, "time": 162.38105652332305}, "c4c94a5527": {"quality": 0.0, "cost": 0.0075986579999999995, "time": 122.43854382038117}, "c4e75ee9ba": {"quality": 0.14946236559139786, "cost": 0.06605840600000001, "time": 123.62291731834412}, "c4f3e7665d": {"quality": 0.04, "cost": 0.004755744000000001, "time": 128.4701939582825}, "c5471bef57": {"quality": 0.0, "cost": 0.034792454, "time": 174.61682531833648}, "c54a408db7": {"quality": 0.11729646697388633, "cost": 0.028817849000000003, "time": 119.10575988292695}, "c59cc41335": {"quality": 0.02, "cost": 0.029414305, "time": 127.53696141242982}, "c5a0b065e0": {"quality": 0.0, "cost": 0.011118656000000001, "time": 86.77302918434142}, "c5a16b834a": {"quality": 0.10623655913978494, "cost": 0.009676605, "time": 163.57068514823914}, "c5fbe2076f": {"quality": 0.09333333333333334, "cost": 0.016622912, "time": 171.14971058368684}, "c617370f6b": {"quality": 0.11134408602150538, "cost": 0.028784580000000004, "time": 123.97298481464387}, "c67f782c7f": {"quality": 0.10301075268817204, "cost": 0.031540366, "time": 122.78840267658234}, "c691a29c42": {"quality": 0.0696774193548387, "cost": 0.015512881999999999, "time": 156.41156446933746}, "c6a339987c": {"quality": 0.0, "cost": 0.009196331000000002, "time": 160.24627866744996}, "c772ff3704": {"quality": 0.06, "cost": 0.008646966, "time": 149.87581686973573}, "c7e3f348c2": {"quality": 0.10946236559139785, "cost": 0.03753916200000001, "time": 149.64094378948212}, "c823589ab6": {"quality": 0.03333333333333333, "cost": 0.030336296, "time": 124.21261048316956}, "c82f834e85": {"quality": 0.11591397849462365, "cost": 0.007749193, "time": 110.63324115276336}, "c85099881f": {"quality": 0.0032258064516129032, "cost": 0.00896294, "time": 77.65277953147888}, "c935a33384": {"quality": 0.11591397849462365, "cost": 0.013818304, "time": 147.3745896100998}, "ca3177461f": {"quality": 0.07290322580645162, "cost": 0.032201930000000004, "time": 144.48511946201324}, "caa7c0bd6b": {"quality": 0.08623655913978495, "cost": 0.007444665, "time": 109.04434747695923}, "cac6b051e9": {"quality": 0.0696774193548387, "cost": 0.008080419, "time": 151.42419612407684}, "cacb342f64": {"quality": 0.10946236559139785, "cost": 0.04521175200000001, "time": 148.37158913612365}, "cb9948679c": {"quality": 0.03612903225806452, "cost": 0.031414252000000004, "time": 155.25814893245698}, "cbb5eb0e74": {"quality": 0.10623655913978494, "cost": 0.008005419, "time": 159.4877107143402}, "cbc32cbeff": {"quality": 0.07729646697388634, "cost": 0.009278367, "time": 176.14444167613982}, "cbd4461293": {"quality": 0.09655913978494624, "cost": 0.0019236219999999998, "time": 56.308462142944336}, "cbe2318045": {"quality": 0.09602150537634407, "cost": 0.008356266000000001, "time": 120.74416449069977}, "cc886fe337": {"quality": 0.03333333333333333, "cost": 0.006418602, "time": 123.71861448287964}, "cc9a6248a0": {"quality": 0.08301075268817204, "cost": 0.027855250000000005, "time": 40.938735127449036}, "ccb2335b3f": {"quality": 0.08946236559139785, "cost": 0.03473691600000001, "time": 175.81531076431276}, "ccdf03a55b": {"quality": 0.04, "cost": 0.04126820400000002, "time": 147.74624931812286}, "ccf72745c1": {"quality": 0.11268817204301075, "cost": 0.010027356, "time": 131.06223764419556}, "cd1d418732": {"quality": 0.0, "cost": 0.005709352, "time": 72.77172949314118}, "cd23c79db1": {"quality": 0.049677419354838714, "cost": 0.009460574000000001, "time": 154.12733085155486}, "cd64fbfcd9": {"quality": 0.08623655913978495, "cost": 0.0068760869999999995, "time": 149.53058531284333}, "cd85a01e81": {"quality": 0.07333333333333333, "cost": 0.034051955, "time": 182.54483699798584}, "ce4bc5f348": {"quality": 0.0, "cost": 0.003405222, "time": 103.34760620594025}, "ce980cf86f": {"quality": 0.10301075268817204, "cost": 0.0039973019999999995, "time": 133.40581440925598}, "ceae8b8bb9": {"quality": 0.10946236559139785, "cost": 0.020052325000000003, "time": 167.7911945104599}, "cecca90dd2": {"quality": 0.10301075268817204, "cost": 0.004860323999999999, "time": 140.68442809581757}, "cf9538faf0": {"quality": 0.17052227342549925, "cost": 0.00948961, "time": 133.63166942596436}, "cf9d2e224c": {"quality": 0.06, "cost": 0.006049894, "time": 97.01887967586518}, "cfd36f3a8c": {"quality": 0.10301075268817204, "cost": 0.015497412000000002, "time": 141.9499899148941}, "cffa29a6ef": {"quality": 0.12301075268817205, "cost": 0.0005813769999999999, "time": 62.55750501155853}, "d03596c3de": {"quality": 0.10946236559139785, "cost": 0.007728069000000001, "time": 166.9729477405548}, "d07b766487": {"quality": 0.07935483870967741, "cost": 0.03461312800000001, "time": 162.30493545532227}, "d0a0a66d75": {"quality": 0.11268817204301075, "cost": 0.022896756000000004, "time": 80.08134040832519}, "d0ce31134c": {"quality": 0.05591397849462365, "cost": 0.038193279999999996, "time": 161.50320081710817}, "d0f9633442": {"quality": 0.0, "cost": 0.008286096, "time": 140.58964149951936}, "d216eab7d8": {"quality": 0.09268817204301075, "cost": 0.008950572, "time": 116.59772584438323}, "d266c19ac8": {"quality": 0.10301075268817204, "cost": 0.005751029999999999, "time": 161.90357491970062}, "d26a70179a": {"quality": 0.10623655913978494, "cost": 0.03584914900000001, "time": 172.63189315795898}, "d2af24b59e": {"quality": 0.06, "cost": 0.008735434, "time": 124.40694677829742}, "d2f2dd5cd4": {"quality": 0.13623655913978494, "cost": 0.023797876000000003, "time": 131.33156244754792}, "d302278f85": {"quality": 0.00967741935483871, "cost": 0.030764487000000004, "time": 169.61139817237853}, "d37dcaea30": {"quality": 0.10946236559139785, "cost": 0.019287263, "time": 164.99651217460632}, "d3a2d50bd7": {"quality": 0.07333333333333333, "cost": 0.004375113, "time": 134.56950817108154}, "d3d4185487": {"quality": 0.06322580645161291, "cost": 0.033443476, "time": 123.06616494655609}, "d3db4cf84d": {"quality": 0.0, "cost": 0.004757544, "time": 99.5310351371765}, "d402233b53": {"quality": 0.10946236559139785, "cost": 0.022382061, "time": 155.35112550258637}, "d43fafa19e": {"quality": 0.00967741935483871, "cost": 0.004969002, "time": 85.30777862071992}, "d446a75eb7": {"quality": 0.10946236559139785, "cost": 0.045008152, "time": 111.52242555618287}, "d48ead13da": {"quality": 0.02, "cost": 0.00741723, "time": 123.98356997966766}, "d5016f4538": {"quality": 0.0, "cost": 0.00508065, "time": 85.11343963146209}, "d55a3613b0": {"quality": 0.10623655913978494, "cost": 0.041770765, "time": 152.50331590175628}, "d58036ba66": {"quality": 0.10301075268817204, "cost": 0.008145371999999998, "time": 106.38735165596009}, "d5a84c782e": {"quality": 0.10623655913978494, "cost": 0.029261760000000005, "time": 104.74087455272675}, "d5b2eef11c": {"quality": 0.12301075268817205, "cost": 0.02737009, "time": 65.21177852153778}, "d6040140b9": {"quality": 0.10301075268817204, "cost": 0.014017109, "time": 153.3877902984619}, "d65185c1a4": {"quality": 0.10623655913978494, "cost": 0.00525975, "time": 117.94775750637055}, "d667351f33": {"quality": 0.10301075268817204, "cost": 0.035737883, "time": 160.19842824935913}, "d6bd3b66ba": {"quality": 0.04, "cost": 0.004024998, "time": 126.8981255054474}, "d6c4e48eeb": {"quality": 0.10946236559139785, "cost": 0.008899966, "time": 125.08437695503235}, "d6cbf265ee": {"quality": 0.0, "cost": 0.001772622, "time": 80.58833994865418}, "d705447fd7": {"quality": 0.02, "cost": 0.03513882900000001, "time": 158.08452146053315}, "d73a9aab4e": {"quality": 0.0, "cost": 0.003741966, "time": 86.37196826934814}, "d752c30d07": {"quality": 0.10301075268817204, "cost": 0.016376438, "time": 126.43599452972413}, "d782682359": {"quality": 0.12301075268817205, "cost": 0.035046282000000005, "time": 158.43969354629516}, "d7c0972014": {"quality": 0.07333333333333333, "cost": 0.010681736, "time": 93.32859888076783}, "d867525748": {"quality": 0.06301075268817204, "cost": 0.01181131, "time": 120.1785500049591}, "d87eb775da": {"quality": 0.10946236559139785, "cost": 0.008138371, "time": 109.30400257110595}, "d8bab6c09b": {"quality": 0.0696774193548387, "cost": 0.011994469, "time": 164.43309626579287}, "d8bcac36e8": {"quality": 0.03333333333333333, "cost": 0.010924840000000002, "time": 119.18308181762694}, "d8eadc0190": {"quality": 0.10301075268817204, "cost": 0.039214723, "time": 115.4887844324112}, "d96677d8d4": {"quality": 0.10301075268817204, "cost": 0.006749594999999999, "time": 111.51266958713532}, "d98f22270e": {"quality": 0.20301075268817204, "cost": 0.034043790000000004, "time": 121.76555292606355}, "d9e2bb21a3": {"quality": 0.05290322580645161, "cost": 0.005608887, "time": 109.89146270751954}, "da95deeb20": {"quality": 0.10301075268817204, "cost": 0.027880385, "time": 84.15849900245667}, "daaadadcc9": {"quality": 0.11591397849462365, "cost": 0.010063653999999998, "time": 151.10103166103363}, "daf855e065": {"quality": 0.11591397849462365, "cost": 0.006424737, "time": 158.07599256038665}, "db00594832": {"quality": 0.12623655913978493, "cost": 0.037128378000000004, "time": 93.38327257633209}, "db19e677c4": {"quality": 0.12301075268817205, "cost": 0.039896044, "time": 130.70764796733857}, "db3c035639": {"quality": 0.11591397849462365, "cost": 0.036307984, "time": 152.2756203889847}, "db41487005": {"quality": 0.0, "cost": 0.029024420000000002, "time": 115.28090765476227}, "db6a7482fd": {"quality": 0.11333333333333334, "cost": 0.027559395, "time": 72.33171219825743}, "db9060cd27": {"quality": 0.0696774193548387, "cost": 0.001964664, "time": 64.76092283725738}, "dbce95a072": {"quality": 0.10301075268817204, "cost": 0.007502654000000001, "time": 103.14728391170502}, "dc195abe5e": {"quality": 0.10623655913978494, "cost": 0.030985093999999998, "time": 136.5469225883484}, "dc3f4b7138": {"quality": 0.02, "cost": 0.028686388000000007, "time": 102.1283097743988}, "dc66bccb1c": {"quality": 0.11456989247311827, "cost": 0.006545776, "time": 108.06170182228088}, "dc90065dea": {"quality": 0.10301075268817204, "cost": 0.010758259999999999, "time": 109.67339315414429}, "dd0d70fedd": {"quality": 0.10301075268817204, "cost": 0.0019252949999999998, "time": 62.47777860164643}, "dddc76b3ca": {"quality": 0.12301075268817205, "cost": 0.03317511000000001, "time": 121.71198806762695}, "de18bf45e1": {"quality": 0.016129032258064516, "cost": 0.001520226, "time": 81.24462885856629}, "de1e56370f": {"quality": 0.10301075268817204, "cost": 0.004404816000000001, "time": 107.52295508384705}, "df2160ecc8": {"quality": 0.10301075268817204, "cost": 0.03259985, "time": 121.08217024803162}, "dfda94bd2a": {"quality": 0.012903225806451613, "cost": 0.001747728, "time": 94.69696862697602}, "dff452a9ca": {"quality": 0.0, "cost": 0.005642568000000001, "time": 113.53185505867005}, "e09f75d9d6": {"quality": 0.12301075268817205, "cost": 0.039859539, "time": 121.64791254997255}, "e0b6a99753": {"quality": 0.10301075268817204, "cost": 0.008541190000000002, "time": 52.27649214267731}, "e0cf6587a7": {"quality": 0.07612903225806453, "cost": 0.035429448, "time": 154.11263673305513}, "e0de4a5929": {"quality": 0.11591397849462365, "cost": 0.03794565500000001, "time": 146.6949642419815}, "e1356fb426": {"quality": 0.10301075268817204, "cost": 0.03224261, "time": 113.18122699260712}, "e20ba014a1": {"quality": 0.10623655913978494, "cost": 0.010716353000000001, "time": 146.11545753479004}, "e21806e3bc": {"quality": 0.06946236559139785, "cost": 0.011148466, "time": 108.38995244503022}, "e24e97564c": {"quality": 0.10623655913978494, "cost": 0.034947423000000005, "time": 141.45279150009156}, "e2673c1ec8": {"quality": 0.10301075268817204, "cost": 0.035187995, "time": 133.8898230791092}, "e26c7bfbdb": {"quality": 0.08301075268817204, "cost": 0.007581762000000001, "time": 99.8463338136673}, "e2f9980b06": {"quality": 0.11591397849462365, "cost": 0.019417651, "time": 133.10050780773162}, "e3445f7632": {"quality": 0.10623655913978494, "cost": 0.004820125, "time": 63.44422569274902}, "e35e5f81a7": {"quality": 0.0, "cost": 0.006810027, "time": 115.5727329492569}, "e376ac53e7": {"quality": 0.08946236559139785, "cost": 0.024290850000000003, "time": 110.75653586387634}, "e3d8bb56da": {"quality": 0.10301075268817204, "cost": 0.021544511000000002, "time": 116.27818155288696}, "e3df4cf041": {"quality": 0.0, "cost": 0.007794071, "time": 129.81809906959535}, "e47dc3abca": {"quality": 0.10623655913978494, "cost": 0.011515712, "time": 132.7535306930542}, "e4b9d4fb41": {"quality": 0.02, "cost": 0.007862585, "time": 139.48534836769102}, "e510bda989": {"quality": 0.13268817204301075, "cost": 0.005762856, "time": 33.74014439582825}, "e517cd2222": {"quality": 0.08761904761904762, "cost": 0.001565182, "time": 53.67046070098877}, "e51b01f418": {"quality": 0.07290322580645162, "cost": 0.005801265, "time": 138.57358028888703}, "e520dfae5b": {"quality": 0.10946236559139785, "cost": 0.006193371, "time": 176.72269680500028}, "e521c9b7e4": {"quality": 0.12301075268817205, "cost": 0.034744782, "time": 125.99484462738036}, "e54097ad5d": {"quality": 0.10301075268817204, "cost": 0.030318109000000006, "time": 143.14947426319122}, "e56a16ca66": {"quality": 0.10623655913978494, "cost": 0.0019251749999999997, "time": 100.41836931705475}, "e5c4abf7ce": {"quality": 0.10301075268817204, "cost": 0.03347102800000001, "time": 170.73657705783845}, "e5d4689312": {"quality": 0.0, "cost": 0.03450822800000001, "time": 180.7019939184189}, "e62a7b27ae": {"quality": 0.049677419354838714, "cost": 0.026221878, "time": 142.64135072231295}, "e6a7aff3bc": {"quality": 0.0, "cost": 0.032406869000000005, "time": 182.2341136932373}, "e736999157": {"quality": 0.04, "cost": 0.031729251, "time": 175.4835301399231}, "e7517a8ce0": {"quality": 0.12301075268817205, "cost": 0.034507195000000004, "time": 118.30208179950714}, "e7520ca5ac": {"quality": 0.12301075268817205, "cost": 0.03356782800000001, "time": 164.09410247802734}, "e7e94ab7a5": {"quality": 0.0, "cost": 0.001703792, "time": 94.30426328182222}, "e887ddf5cc": {"quality": 0.10623655913978494, "cost": 0.041667262, "time": 169.6624465703964}, "e94fb5a295": {"quality": 0.0, "cost": 0.033384014000000004, "time": 134.82464711666108}, "ea6ecc5653": {"quality": 0.06301075268817204, "cost": 0.007951587, "time": 134.84457714557647}, "ea8bcb3ae2": {"quality": 0.10301075268817204, "cost": 0.017331892, "time": 134.0905973672867}, "ebbe8b6c4f": {"quality": 0.0, "cost": 0.007050224, "time": 106.51247243881225}, "ebdf3abff2": {"quality": 0.10301075268817204, "cost": 0.013816473000000001, "time": 136.48189253807067}, "ec55dba809": {"quality": 0.10301075268817204, "cost": 0.03585076400000001, "time": 154.6976619243622}, "ecb5f78f37": {"quality": 0.0, "cost": 0.011340265, "time": 174.9997961997986}, "ecda3d74cb": {"quality": 0.02, "cost": 0.032141616, "time": 116.44101283550262}, "ece10c0388": {"quality": 0.18301075268817205, "cost": 0.05883078800000001, "time": 113.18113870620726}, "ed6b5480a5": {"quality": 0.06946236559139785, "cost": 0.0049282679999999995, "time": 77.08623633384704}, "eda630dc85": {"quality": 0.10623655913978494, "cost": 0.007813983, "time": 120.82467505931854}, "edaaee5ed4": {"quality": 0.08946236559139785, "cost": 0.006681024000000001, "time": 74.40945067405701}, "edb2b764aa": {"quality": 0.02, "cost": 0.013618342, "time": 113.58856484889984}, "edc52339db": {"quality": 0.03333333333333333, "cost": 0.013122086000000002, "time": 151.4774285554886}, "ede7071775": {"quality": 0.12301075268817205, "cost": 0.059439179999999994, "time": 100.97825186252595}, "ee46042c5d": {"quality": 0.0, "cost": 0.01214812, "time": 152.4212220430374}, "ee68c51f73": {"quality": 0.12301075268817205, "cost": 0.030089499999999998, "time": 61.54816117286683}, "ee7b726747": {"quality": 0.11591397849462365, "cost": 0.017031875000000002, "time": 160.18388693332673}, "eec5f32da9": {"quality": 0.11015360983102919, "cost": 0.040522985000000004, "time": 149.7510376214981}, "eed40e4378": {"quality": 0.10623655913978494, "cost": 0.03209097500000001, "time": 152.84840388298034}, "eef12d478b": {"quality": 0.08258064516129032, "cost": 0.0049949880000000006, "time": 113.2755955696106}, "ef37b3e0be": {"quality": 0.0, "cost": 0.009670806, "time": 178.49481089115142}, "ef43d497f1": {"quality": 0.08301075268817204, "cost": 0.012532439999999999, "time": 159.2441192626953}, "ef4d4c4a62": {"quality": 0.08623655913978495, "cost": 0.00850348, "time": 126.41268122196198}, "ef9a651425": {"quality": 0.11591397849462365, "cost": 0.036512095, "time": 164.66946573257445}, "f0655621af": {"quality": 0.02967741935483871, "cost": 0.0047729999999999995, "time": 45.447355723381044}, "f076b4c9ae": {"quality": 0.10623655913978494, "cost": 0.007286825, "time": 118.35513505935668}, "f11eddb4ed": {"quality": 0.10301075268817204, "cost": 0.012131963000000003, "time": 151.4715585231781}, "f1408da253": {"quality": 0.10301075268817204, "cost": 0.009555081, "time": 155.64059100151061}, "f18cf41929": {"quality": 0.0, "cost": 0.003205638, "time": 99.03487899303437}, "f1aa0b0b42": {"quality": 0.10946236559139785, "cost": 0.025626688, "time": 115.38884313106537}, "f1bda127f6": {"quality": 0.10301075268817204, "cost": 0.012302958, "time": 102.7573136806488}, "f1f373e58e": {"quality": 0.10623655913978494, "cost": 0.039605126000000004, "time": 118.39543704986573}, "f2a2e91541": {"quality": 0.09660522273425498, "cost": 0.008657139000000001, "time": 145.9810749053955}, "f2c04ed1c8": {"quality": 0.10301075268817204, "cost": 0.004977713999999999, "time": 68.23735935688019}, "f2cf5db12d": {"quality": 0.09268817204301075, "cost": 0.008181425999999999, "time": 120.30662734508515}, "f366c0dd10": {"quality": 0.00967741935483871, "cost": 0.029196041000000002, "time": 114.87482616901397}, "f4303a5b4f": {"quality": 0.10301075268817204, "cost": 0.041115134000000005, "time": 145.90202000141142}, "f437481e3b": {"quality": 0.21591397849462365, "cost": 0.012506865999999998, "time": 151.16193616390228}, "f4bc6b63a7": {"quality": 0.10301075268817204, "cost": 0.05883107500000001, "time": 110.05861134529113}, "f4dc556633": {"quality": 0.032903225806451615, "cost": 0.035164677000000005, "time": 165.65216555595396}, "f4deb72db6": {"quality": 0.0, "cost": 0.00779049, "time": 124.83286740779877}, "f4ef2b9c33": {"quality": 0.08623655913978495, "cost": 0.025671540000000003, "time": 110.72261786460876}, "f566d6d6a1": {"quality": 0.12301075268817205, "cost": 0.030296474, "time": 118.43880999088287}, "f5b9a94dcc": {"quality": 0.0, "cost": 0.00286785, "time": 96.0643517255783}, "f5c27e7172": {"quality": 0.029677419354838707, "cost": 0.003395451, "time": 131.81098673343658}, "f5e53d963b": {"quality": 0.11623655913978495, "cost": 0.0022498320000000002, "time": 82.10175185203552}, "f614235c15": {"quality": 0.0, "cost": 0.0015151140000000001, "time": 93.65864548683166}, "f6546149e3": {"quality": 0.10623655913978494, "cost": 0.034531665, "time": 126.04132678508759}, "f74ec023e4": {"quality": 0.11591397849462365, "cost": 0.011997618, "time": 167.2357141494751}, "f7b048bd54": {"quality": 0.10301075268817204, "cost": 0.008602385, "time": 132.47917804718017}, "f7c4df993e": {"quality": 0.10301075268817204, "cost": 0.007398474, "time": 140.5795171499252}, "f854533145": {"quality": 0.10301075268817204, "cost": 0.010546359, "time": 164.53556537628174}, "f89b8a1930": {"quality": 0.10946236559139785, "cost": 0.016153075, "time": 165.33965291976926}, "f93d9a2693": {"quality": 0.06301075268817204, "cost": 0.006399356999999998, "time": 130.61231911182404}, "f97d91a249": {"quality": 0.13591397849462367, "cost": 0.029466680000000002, "time": 128.38061220645903}, "f99096d89c": {"quality": 0.11268817204301075, "cost": 0.009063081, "time": 164.117862033844}, "f9e8e221f3": {"quality": 0.07729646697388634, "cost": 0.007258314, "time": 137.73151681423187}, "fa38879eab": {"quality": 0.07333333333333333, "cost": 0.013395525000000002, "time": 171.87790231704713}, "fa71111570": {"quality": 0.09333333333333334, "cost": 0.033387521, "time": 163.68935594558718}, "fa7882d46b": {"quality": 0.11591397849462365, "cost": 0.01214556, "time": 168.27967350482942}, "fa906520d1": {"quality": 0.10301075268817204, "cost": 0.012420450999999999, "time": 173.23661334514617}, "faabebaa30": {"quality": 0.08301075268817204, "cost": 0.01007883, "time": 177.29549469947813}, "fb0339a7d0": {"quality": 0.10623655913978494, "cost": 0.012603655000000002, "time": 175.26443803310394}, "fb216ad6b3": {"quality": 0.08623655913978495, "cost": 0.0027521299999999998, "time": 48.80933380126953}, "fb6216880a": {"quality": 0.04, "cost": 0.010161849, "time": 176.34763963222503}, "fba499b89d": {"quality": 0.03333333333333333, "cost": 0.034680020000000006, "time": 176.12596268653868}, "fbc010a368": {"quality": 0.0696774193548387, "cost": 0.03236535, "time": 140.89571504592897}, "fbc02e2e07": {"quality": 0.10946236559139785, "cost": 0.032320148, "time": 134.5744981765747}, "fbd6c45271": {"quality": 0.11591397849462365, "cost": 0.010223524000000001, "time": 171.81036722660065}, "fc0a156e16": {"quality": 0.10623655913978494, "cost": 0.014270385, "time": 127.52731266021729}, "fc1fd5bf54": {"quality": 0.0701536098310292, "cost": 0.005700248999999999, "time": 163.72950367927552}, "fc6967a75b": {"quality": 0.0, "cost": 0.03203869000000001, "time": 176.3084951877594}, "fc73c3b0fa": {"quality": 0.12301075268817205, "cost": 0.011811562000000001, "time": 53.39196453094482}, "fce38334b2": {"quality": 0.03612903225806452, "cost": 0.013067388, "time": 160.21355810165403}, "fce5fca128": {"quality": 0.15946236559139787, "cost": 0.038042624000000004, "time": 158.27728819847107}, "fd0709359e": {"quality": 0.10301075268817204, "cost": 0.0006582199999999999, "time": 41.16247012615204}, "fd1f809d64": {"quality": 0.10301075268817204, "cost": 0.040954353, "time": 135.35224254131316}, "fd2c994a9d": {"quality": 0.10946236559139785, "cost": 0.031750883, "time": 155.9396536588669}, "fddccfbf94": {"quality": 0.20301075268817204, "cost": 0.010760402, "time": 121.92865846157073}, "fe7fa741b4": {"quality": 0.11134408602150538, "cost": 0.016085802000000003, "time": 126.83495450019836}, "fe9e1fec71": {"quality": 0.12301075268817205, "cost": 0.033332814, "time": 128.354682469368}, "fea4734c09": {"quality": 0.12301075268817205, "cost": 0.005711668, "time": 87.13299584388733}, "fef1ca27fa": {"quality": 0.07913978494623655, "cost": 0.013074829999999999, "time": 140.50196795463563}, "ff11cb6a7a": {"quality": 0.08301075268817204, "cost": 0.012798740000000001, "time": 135.25200283527374}, "ff171e34e2": {"quality": 0.0, "cost": 0.004113531, "time": 106.45833990573882}, "ff1c958e21": {"quality": 0.0, "cost": 0.0030646650000000003, "time": 87.37032148838043}, "ff8df4ace9": {"quality": 0.08, "cost": 0.032799690000000006, "time": 86.2204920053482}, "ff8e68049a": {"quality": 0.10301075268817204, "cost": 0.006420515999999999, "time": 71.03863010406494}} ================================================ FILE: abacus-research/biodex-revision-priors-maxquality.json ================================================ {"00b02360ef": {"quality": 0.21648684648684646, "cost": 0.040317356500000005, "time": 40.99749290347099}, "025a41642e": {"quality": 0.2323391192141192, "cost": 0.011047775499999999, "time": 38.92555040717125}, "03c4bc9bb0": {"quality": 0.25034264346764346, "cost": 0.011205506, "time": 22.393556386232376}, "04ed954ae9": {"quality": 0.25250624375624375, "cost": 0.04878718750000001, "time": 20.034630668163302}, "059dee8af9": {"quality": 0.2151956376956377, "cost": 0.00895401925, "time": 31.44526737332344}, "05fde83b5e": {"quality": 0.20102897102897105, "cost": 0.007020948000000001, "time": 27.97608198523521}, "06b62ff472": {"quality": 0.26966144966144967, "cost": 0.005215445, "time": 14.247572779655457}, "0730a1221a": {"quality": 0.25292291042291043, "cost": 0.027662595, "time": 33.58496024608612}, "07aec969dd": {"quality": 0.2148887223887224, "cost": 0.0008298215000000001, "time": 43.21092504262924}, "0a11f0ae2e": {"quality": 0.24079150016650014, "cost": 0.010170454500000002, "time": 20.595997565984725}, "0aeb79b24c": {"quality": 0.14777319902319902, "cost": 0.0058596425, "time": 7.766067373752594}, "0b3eafcb23": {"quality": 0.1913265900765901, "cost": 0.0013463715, "time": 19.126238137483597}, "0bc5bbe9f8": {"quality": 0.19658452658452658, "cost": 0.0314150555, "time": 42.02523586153984}, "0c328ddadf": {"quality": 0.1950774919524919, "cost": 0.007442179, "time": 22.943194967508315}, "0db4d3e16d": {"quality": 0.18679285991785993, "cost": 0.0032097465, "time": 19.68699924945831}, "0df97ab2f2": {"quality": 0.22131729381729381, "cost": 0.008563199, "time": 43.75024158954621}, "0e0162da5c": {"quality": 0.21570068820068822, "cost": 0.027131577500000004, "time": 33.71510918140412}, "0e0216352f": {"quality": 0.16231865356865358, "cost": 0.009209249, "time": 33.987769949436185}, "0ecd21b93a": {"quality": 0.2128450715950716, "cost": 0.027050046, "time": 27.799335622787474}, "0ff2bde030": {"quality": 0.2117928599178599, "cost": 0.033819734000000004, "time": 19.09183721542358}, "1059f21a6c": {"quality": 0.21909507159507158, "cost": 0.0006545234999999999, "time": 13.647678703069687}, "112b665311": {"quality": 0.21884552947052946, "cost": 0.004884652, "time": 22.208240193128585}, "11db1dbc4f": {"quality": 0.1719506882006882, "cost": 0.003233838, "time": 22.85480940937996}, "13145c10ec": {"quality": 0.23070159007659008, "cost": 0.0313856595, "time": 35.557817763090135}, "147915874a": {"quality": 0.2280723443223443, "cost": 0.00055762025, "time": 34.335060054063796}, "16e1e2aca6": {"quality": 0.25766650016650017, "cost": 0.005274602500000001, "time": 13.559284949302672}, "1914dfad94": {"quality": 0.2624284049284049, "cost": 0.02536484, "time": 25.639233177900316}, "1ab9dcd7d6": {"quality": 0.2867339604839605, "cost": 0.0010822935, "time": 18.924210703372957}, "1dc749411d": {"quality": 0.2199178599178599, "cost": 0.010029340000000001, "time": 14.274628734588624}, "1dd84b0308": {"quality": 0.2286371961371961, "cost": 0.027319404999999998, "time": 25.991028892993928}, "1dd9d9bef2": {"quality": 0.2226475607725608, "cost": 0.007434026250000001, "time": 31.69112855195999}, "1f0ba92b8b": {"quality": 0.22614052614052613, "cost": 0.0175957475, "time": 36.91505531668663}, "1f1b5a019d": {"quality": 0.2593331668331669, "cost": 0.033523670000000005, "time": 26.298819702863693}, "1f41a35dd3": {"quality": 0.14541666666666667, "cost": 0.004300389, "time": 17.449908739328386}, "212c3fccbf": {"quality": 0.2939645770895771, "cost": 0.0327319625, "time": 27.72252732515335}, "219d51cd03": {"quality": 0.1703336247086247, "cost": 0.0009408769999999999, "time": 18.91644185781479}, "238ed27a56": {"quality": 0.24007659007659007, "cost": 0.0295427895, "time": 31.712434768676758}, "23f3d97d47": {"quality": 0.23146457708957707, "cost": 0.02700624675, "time": 32.10481671690941}, "253547264d": {"quality": 0.1917432567432567, "cost": 0.026901585, "time": 23.115290719270703}, "27e14a0973": {"quality": 0.23250624375624376, "cost": 0.033178285, "time": 36.78056996464729}, "29960739d3": {"quality": 0.221773643023643, "cost": 0.032106853000000005, "time": 29.176937651634216}, "2b0a8d5614": {"quality": 0.2043530081030081, "cost": 0.022987428499999997, "time": 20.08997523188591}, "2ccd104a68": {"quality": 0.20102897102897105, "cost": 0.0184998355, "time": 45.19519582390785}, "2d1b0cd5d9": {"quality": 0.22214008214008213, "cost": 0.0059833375, "time": 18.541752833127973}, "2dcffa3d30": {"quality": 0.27596042846042845, "cost": 0.009521012499999999, "time": 26.920003366470336}, "2ebe033e6a": {"quality": 0.21273532023532027, "cost": 0.0073467315, "time": 21.433203184604643}, "31e8ef9410": {"quality": 0.10839285714285715, "cost": 0.0007289739999999999, "time": 63.23130738735199}, "32a628c971": {"quality": 0.25402500277500273, "cost": 0.037233726, "time": 26.05542423725128}, "336e1a4fdc": {"quality": 0.1680573593073593, "cost": 0.031870938, "time": 38.903397005796435}, "33e0fdf4e1": {"quality": 0.2249284049284049, "cost": 0.005128618, "time": 22.448986053466797}, "38a60c768a": {"quality": 0.23622648185148185, "cost": 0.027102593, "time": 38.203392094373704}, "39bb54fbfb": {"quality": 0.248528971028971, "cost": 0.03168909524999999, "time": 49.03234923481941}, "3ace395562": {"quality": 0.20436230436230435, "cost": 0.025813554500000002, "time": 24.070003932714464}, "3bbe08bf63": {"quality": 0.2427335164835165, "cost": 0.033579119000000004, "time": 43.73568968772888}, "3caf8d77c5": {"quality": 0.25665397102897103, "cost": 0.0361807135, "time": 33.51353464722634}, "3d25abc0cb": {"quality": 0.2298871961371961, "cost": 0.008877388500000001, "time": 38.62544178366661}, "3e19290ef5": {"quality": 0.2251261932511933, "cost": 0.026017763000000006, "time": 31.622546792030334}, "3e8e089a02": {"quality": 0.232903971028971, "cost": 0.0243675525, "time": 12.379487264156342}, "3f1771caee": {"quality": 0.1291142884892885, "cost": 0.003903147, "time": 22.171902561187743}, "3f7beb53c4": {"quality": 0.26968038905538905, "cost": 0.00175834875, "time": 17.588406985998155}, "4081f36f1d": {"quality": 0.20352897102897105, "cost": 0.009008155, "time": 35.7961721599102}, "410edf5f69": {"quality": 0.24408452658452662, "cost": 0.033720045500000004, "time": 39.4614034473896}, "44ec78f301": {"quality": 0.2112887112887113, "cost": 0.0312310875, "time": 28.79513317346573}, "44eeb11408": {"quality": 0.23811230436230435, "cost": 0.027474213500000004, "time": 33.9330511868}, "45623e6def": {"quality": 0.27609841547341546, "cost": 0.0343063295, "time": 33.21763527989388}, "470749179f": {"quality": 0.22933316683316685, "cost": 0.0050279755, "time": 24.99732105731964}, "47ea56eba0": {"quality": 0.19954240204240203, "cost": 0.018521998, "time": 30.92959639430046}, "49fc8b3768": {"quality": 0.16649433899433902, "cost": 0.0028119045000000002, "time": 17.623043191432952}, "4a60adbf47": {"quality": 0.18713078588078588, "cost": 0.013557406999999999, "time": 28.078510254621506}, "4ba372d292": {"quality": 0.24674478299478303, "cost": 0.007506609250000001, "time": 32.822478234767914}, "4bf38b1922": {"quality": 0.19991785991785993, "cost": 0.031763846750000005, "time": 40.722243314981455}, "4c415b4b86": {"quality": 0.1526658757908758, "cost": 0.025909743, "time": 14.397501182556152}, "4d1fc31b49": {"quality": 0.2615845265845266, "cost": 0.024915399, "time": 18.815959566831587}, "4d8c84f53a": {"quality": 0.2150567488067488, "cost": 0.0271872135, "time": 31.158079880476}, "4df2245b9b": {"quality": 0.14843878343878342, "cost": 0.0011828115, "time": 5.0212810933589935}, "4e9274d39b": {"quality": 0.2178261322011322, "cost": 0.0059426750000000006, "time": 16.452285903692246}, "4f87ed7c2e": {"quality": 0.23269563769563767, "cost": 0.0090589945, "time": 26.12176838517189}, "51a340bebc": {"quality": 0.23375624375624376, "cost": 0.032705517, "time": 28.51812062859535}, "52dee74e25": {"quality": 0.2215728021978022, "cost": 0.032394368, "time": 31.705753284692765}, "533fddb3bd": {"quality": 0.13879745254745254, "cost": 0.0016485569999999997, "time": 9.408051878213882}, "5358cb5855": {"quality": 0.18664467476967478, "cost": 0.027746372000000002, "time": 23.547580116987227}, "53b6c3e00f": {"quality": 0.2590950715950716, "cost": 0.000778896, "time": 12.53750283718109}, "5400c14640": {"quality": 0.2697979104229104, "cost": 0.029639904999999994, "time": 41.52563037276268}, "55e373a7e2": {"quality": 0.16850885225885226, "cost": 0.0406613025, "time": 22.814987272024155}, "5652070ecb": {"quality": 0.17858849483849482, "cost": 0.0075221129999999995, "time": 32.25966700911522}, "56c5cbc75a": {"quality": 0.23478951603951606, "cost": 0.000632159, "time": 38.25331119894982}, "577b168d21": {"quality": 0.2182725607725608, "cost": 0.00705232, "time": 13.326130753755569}, "57cabf0e4b": {"quality": 0.22299325674325673, "cost": 0.02575083125, "time": 24.347850561141968}, "58dbfc0499": {"quality": 0.2620895770895771, "cost": 0.055131649000000005, "time": 28.987264293432233}, "5a5739c3d2": {"quality": 0.20117597680097682, "cost": 0.007228886499999999, "time": 25.403514659404756}, "5b12979e52": {"quality": 0.18208513708513707, "cost": 0.0038426475, "time": 22.1737478017807}, "5b384da665": {"quality": 0.20099920912420913, "cost": 0.014891756499999999, "time": 28.11651138663292}, "5b837baf2e": {"quality": 0.23083957708957709, "cost": 0.04059063350000001, "time": 38.7564038336277}, "5c8acd9a75": {"quality": 0.1964952408702409, "cost": 0.027179780499999997, "time": 31.81220202445984}, "5da68874af": {"quality": 0.1768232461982462, "cost": 0.00942769325, "time": 30.90108055472374}, "5e304846b6": {"quality": 0.13062375124875125, "cost": 0.00801055675, "time": 21.377113670110703}, "5e608d2a8b": {"quality": 0.160135212010212, "cost": 0.0008863050000000001, "time": 21.419854152202603}, "5e6ad9fb8a": {"quality": 0.19750624375624376, "cost": 0.025521787500000004, "time": 20.569793158769606}, "5ec5b185c9": {"quality": 0.18449037074037078, "cost": 0.002478051, "time": 34.97374950647354}, "608e332141": {"quality": 0.24616785991785994, "cost": 0.03769494400000001, "time": 25.56632845401764}, "610753ffe5": {"quality": 0.2546202408702409, "cost": 0.0301900835, "time": 22.38964868783951}, "614e59c7e7": {"quality": 0.22919427794427794, "cost": 0.0017370554999999999, "time": 16.915371453762056}, "62320710f0": {"quality": 0.25123730436230435, "cost": 0.03658336050000001, "time": 31.835360258817673}, "649df27d73": {"quality": 0.23370664058164056, "cost": 0.033310952000000005, "time": 22.02554016113281}, "64f956ba72": {"quality": 0.2440845265845266, "cost": 0.028164125, "time": 15.42359745502472}, "65657a3d04": {"quality": 0.21516650016650019, "cost": 0.011299091, "time": 48.3486190378666}, "6604abbd43": {"quality": 0.24808316683316683, "cost": 0.026956945250000003, "time": 36.97830517292023}, "67050fa89b": {"quality": 0.18341144966144968, "cost": 0.0313915525, "time": 39.6457058608532}, "67f05c5985": {"quality": 0.22079150016650018, "cost": 0.02690715625, "time": 22.91822772026062}, "67fe27bce0": {"quality": 0.24436230436230436, "cost": 0.041374972, "time": 33.75530356168747}, "698d1b1d0e": {"quality": 0.25665397102897103, "cost": 0.04129188500000001, "time": 50.97259179949761}, "69e8aa9f73": {"quality": 0.1903450715950716, "cost": 0.00740056, "time": 34.38110066056252}, "6b240a8971": {"quality": 0.21965180652680652, "cost": 0.0090931325, "time": 28.12567190527916}, "6b56430005": {"quality": 0.23343968531468534, "cost": 0.027798901, "time": 43.157285338640214}, "6b611b7193": {"quality": 0.2807205294705295, "cost": 0.06928643750000002, "time": 21.161213809251784}, "6b77ef93d3": {"quality": 0.1493911643911644, "cost": 0.0015440895, "time": 12.580836659669876}, "6beefce17b": {"quality": 0.27009552947052945, "cost": 0.03215786600000001, "time": 23.689218693971632}, "6cb7c9a802": {"quality": 0.27246836496836496, "cost": 0.0316063565, "time": 41.612704527378085}, "6cba1da5b3": {"quality": 0.18273684648684646, "cost": 0.013787275500000001, "time": 24.630589830875394}, "6dea54bf99": {"quality": 0.2606321456321456, "cost": 0.004957138, "time": 21.090058833360672}, "6eda6f7780": {"quality": 0.25468038905538903, "cost": 0.011272949, "time": 26.375605469942094}, "708f3ca5da": {"quality": 0.12009552947052946, "cost": 0.0015416722499999999, "time": 13.809926038980484}, "72d75a5f01": {"quality": 0.23014513264513264, "cost": 0.0153489635, "time": 28.418202906847}, "72d99dee93": {"quality": 0.2350756882006882, "cost": 0.029338768500000008, "time": 44.55438640117645}, "73272e5bd3": {"quality": 0.2751956376956377, "cost": 0.0333353575, "time": 29.94201866388321}, "74608570f5": {"quality": 0.11613782051282051, "cost": 0.0005538545, "time": 27.12369663715363}, "7531840182": {"quality": 0.1604120879120879, "cost": 0.0019984210000000002, "time": 58.24117745161056}, "77ba656673": {"quality": 0.18522602397602397, "cost": 0.008878578250000001, "time": 24.636592674255372}, "77fe41f02f": {"quality": 0.1971118464868465, "cost": 0.04380676650000001, "time": 36.06331704854965}, "7857e95756": {"quality": 0.2174590687090687, "cost": 0.014965390000000002, "time": 26.90245589017868}, "78984e94f0": {"quality": 0.21357017982017984, "cost": 0.007025722, "time": 20.596780753135683}, "7986633543": {"quality": 0.20898684648684648, "cost": 0.00747764175, "time": 33.76270450353623}, "7bdbc32b57": {"quality": 0.27164314851814847, "cost": 0.04712992375, "time": 27.709715336561203}, "7c89cff787": {"quality": 0.15383470695970697, "cost": 0.0016299584999999999, "time": 19.46316229701042}, "7dc51ec191": {"quality": 0.2680617993117993, "cost": 0.041678240500000005, "time": 52.64214360117913}, "7e4a6245ac": {"quality": 0.19536401098901102, "cost": 0.00056989225, "time": 64.66312664747238}, "7ebc55ebcf": {"quality": 0.26186230436230434, "cost": 0.0089994125, "time": 49.139087688922885}, "7f14114a48": {"quality": 0.20424325674325672, "cost": 0.0130888145, "time": 24.788931846618652}, "7fb587503e": {"quality": 0.2053228021978022, "cost": 0.009328153000000002, "time": 19.42089232802391}, "7ff82b6f8d": {"quality": 0.20713474025974027, "cost": 0.029847356000000005, "time": 21.508908247947694}, "806a5ef096": {"quality": 0.22686230436230437, "cost": 0.024261581, "time": 14.832676833868026}, "8156d78e42": {"quality": 0.1876665001665002, "cost": 0.007510368999999999, "time": 39.953574234247206}, "815cfb848c": {"quality": 0.24188124375624376, "cost": 0.02595560125, "time": 23.38996571302414}, "820b42e0b1": {"quality": 0.22338078588078586, "cost": 0.0027539085000000004, "time": 21.940480226278304}, "82a733626e": {"quality": 0.24082063769563766, "cost": 0.029802983, "time": 24.7150899887085}, "83d6300ced": {"quality": 0.22958957708957708, "cost": 0.009135905500000001, "time": 44.96203254461288}, "844c822d56": {"quality": 0.19127851315351316, "cost": 0.0269840145, "time": 23.020123237371443}, "84f9f20f75": {"quality": 0.2272988122988123, "cost": 0.02394343, "time": 14.9634037733078}, "8525986a00": {"quality": 0.2595895770895771, "cost": 0.03125997600000001, "time": 45.89759058356285}, "85db012139": {"quality": 0.28459047896547895, "cost": 0.04838826250000001, "time": 21.917299997806552}, "85ddc8a71e": {"quality": 0.19811230436230437, "cost": 0.00595452, "time": 21.147695326805113}, "8767267bed": {"quality": 0.1887950244200244, "cost": 0.0064469125000000006, "time": 7.573464637994766}, "882238f677": {"quality": 0.1552930402930403, "cost": 0.0032291700000000004, "time": 25.413773769140242}, "88f78bdeaf": {"quality": 0.12028311965811966, "cost": 0.0005936184999999999, "time": 24.23035447001457}, "8af1605300": {"quality": 0.21607017982017981, "cost": 0.007401837500000002, "time": 26.804660773277284}, "8c09cba5e2": {"quality": 0.21450119325119324, "cost": 0.02589227425, "time": 26.046354007720947}, "8e5d966cf6": {"quality": 0.15911401098901098, "cost": 0.0016278529999999999, "time": 76.43126168251038}, "8eacbc7240": {"quality": 0.2039868464868465, "cost": 0.0132405425, "time": 26.23775848150253}, "8fc917c575": {"quality": 0.2181123043623044, "cost": 0.0075467905, "time": 27.82010214924812}, "8fe43ec148": {"quality": 0.24137862137862137, "cost": 0.0031192755, "time": 28.713243955373763}, "900de152da": {"quality": 0.2329229104229104, "cost": 0.028014499999999998, "time": 11.612605232000352}, "901ab65b55": {"quality": 0.2703345265845266, "cost": 0.027957687500000005, "time": 9.624877899885178}, "918182aa18": {"quality": 0.19238629426129428, "cost": 0.002521713, "time": 45.21123977303505}, "91ac37a21a": {"quality": 0.140364010989011, "cost": 0.0072078985000000005, "time": 59.984852832555774}, "9252afa4ec": {"quality": 0.20398532023532023, "cost": 0.026106901500000005, "time": 17.68542433977127}, "939b0c3ed9": {"quality": 0.2067141192141192, "cost": 0.009995370500000001, "time": 20.511219489574433}, "93a705cb0d": {"quality": 0.23775578588078589, "cost": 0.0107520785, "time": 34.44544678330421}, "94091e2968": {"quality": 0.2271605477855478, "cost": 0.011070436, "time": 29.48041752576828}, "945a9565fa": {"quality": 0.19553356365856367, "cost": 0.005568486, "time": 9.69417832493782}, "946baa620a": {"quality": 0.2275567488067488, "cost": 0.025270525000000002, "time": 17.454011952877046}, "950ed976dd": {"quality": 0.1091089466089466, "cost": 0.00364847625, "time": 27.579946410655978}, "955689ee8f": {"quality": 0.18228708791208792, "cost": 0.001921847, "time": 29.050129276514056}, "95c9b1b1dd": {"quality": 0.21440018315018317, "cost": 0.031808489, "time": 30.75392867922783}, "95ebba630d": {"quality": 0.16167443667443665, "cost": 0.0045665385, "time": 21.370410311222074}, "966282f059": {"quality": 0.16554410866910865, "cost": 0.023086932000000004, "time": 19.98903277516365}, "96c401772b": {"quality": 0.24861825674325677, "cost": 0.0028384205, "time": 21.471487557888032}, "9718d880f6": {"quality": 0.1984595265845266, "cost": 0.0316786545, "time": 50.695869612693784}, "971ae914cb": {"quality": 0.2652461427461428, "cost": 0.025022515000000002, "time": 14.41015980243683}, "97566e70dd": {"quality": 0.23030386280386278, "cost": 0.0172863905, "time": 32.660637366771695}, "97c882ca1f": {"quality": 0.22630674880674884, "cost": 0.036967587999999996, "time": 26.217366951704026}, "98e28fbb93": {"quality": 0.1839868464868465, "cost": 0.0100201785, "time": 10.730712401866914}, "992ab25916": {"quality": 0.23011779886779885, "cost": 0.00176559, "time": 17.568689429759978}, "9a271dff29": {"quality": 0.19818986568986569, "cost": 0.0093025505, "time": 29.283438795804976}, "9aa6fd8b03": {"quality": 0.20840992340992343, "cost": 0.0059763937500000005, "time": 14.063047587871552}, "9b12b7d441": {"quality": 0.26338078588078584, "cost": 0.01091874575, "time": 34.633186584711076}, "9bf31a2127": {"quality": 0.20186230436230435, "cost": 0.0335034275, "time": 53.72036165595055}, "9c4ad5aed3": {"quality": 0.21813124375624376, "cost": 0.025749503750000007, "time": 27.629801028966902}, "9cbb34ee5f": {"quality": 0.19157370407370405, "cost": 0.0024265880000000004, "time": 20.018360090255737}, "9e16d7ba9c": {"quality": 0.2621788628038628, "cost": 0.032505795500000004, "time": 34.68335065841675}, "9e1a7dd196": {"quality": 0.19222097347097347, "cost": 0.003654598499999999, "time": 18.135820919275282}, "9e6775ac33": {"quality": 0.1184478021978022, "cost": 0.0016337835, "time": 15.268957149982452}, "9e8674b6c7": {"quality": 0.2206123043623044, "cost": 0.033377518, "time": 39.97827153205871}, "9ea12d8d80": {"quality": 0.15865613553113553, "cost": 0.0066404640000000004, "time": 11.089978641271593}, "a232f1dfb6": {"quality": 0.23243922743922746, "cost": 0.0017390729999999998, "time": 17.556494808197023}, "a3b7cb6c33": {"quality": 0.22440018315018312, "cost": 0.0092955765, "time": 25.77978389263153}, "a72f24f687": {"quality": 0.22096764346764347, "cost": 0.008759131000000002, "time": 17.248067778348922}, "a8aac20fd9": {"quality": 0.24810057997558, "cost": 0.028210719500000002, "time": 20.938565105199814}, "a9e811d4a7": {"quality": 0.2364456376956377, "cost": 0.024409971, "time": 15.286300283670425}, "aa1fc47a86": {"quality": 0.20214008214008214, "cost": 0.02572883575, "time": 27.581201010942458}, "aa67b102e7": {"quality": 0.23873730436230436, "cost": 0.030387160750000003, "time": 30.50166518688202}, "ad453a813f": {"quality": 0.21407370407370407, "cost": 0.003257968, "time": 14.862909889221193}, "add25d67a0": {"quality": 0.21374875124875123, "cost": 0.028111017000000002, "time": 24.08863323330879}, "adf6ae1ba7": {"quality": 0.16509552947052947, "cost": 0.0030274415000000002, "time": 9.59496791958809}, "af57afe626": {"quality": 0.238234681984682, "cost": 0.0028139635, "time": 19.57368974685669}, "b07ae35700": {"quality": 0.220353465978466, "cost": 0.003183216, "time": 24.758755880594254}, "b09741c8f7": {"quality": 0.1432112332112332, "cost": 0.0005155585, "time": 36.59446266293526}, "b1b81c0847": {"quality": 0.2774873043623044, "cost": 0.024767306250000003, "time": 17.89712796807289}, "b38fbeda99": {"quality": 0.19523684648684647, "cost": 0.00822793925, "time": 30.255630522966385}, "b3b92b1835": {"quality": 0.26213078588078587, "cost": 0.04786767, "time": 23.704857051372528}, "b3bba3eee2": {"quality": 0.1911576617826618, "cost": 0.009401211, "time": 32.60036996603012}, "b40727bd68": {"quality": 0.2542234154734155, "cost": 0.025418039000000003, "time": 17.466598081588742}, "b6a0f78896": {"quality": 0.23056179931179932, "cost": 0.008894759499999998, "time": 32.4968527674675}, "b84e122880": {"quality": 0.24589008214008212, "cost": 0.008891399000000001, "time": 39.84640188813209}, "b8c6c44a39": {"quality": 0.24308316683316683, "cost": 0.029864208750000003, "time": 21.440666025877}, "bb9ed9dffe": {"quality": 0.15225163725163726, "cost": 0.0015893925, "time": 19.59902848601341}, "bbc50a0411": {"quality": 0.1945796564546564, "cost": 0.0031702530000000005, "time": 33.99009187817573}, "bbd0498616": {"quality": 0.22329150016650018, "cost": 0.05419105850000001, "time": 20.79185708165169}, "bd018ec9d7": {"quality": 0.20710851648351647, "cost": 0.0006870085, "time": 58.566940271854406}, "be11a8a86e": {"quality": 0.22035589410589412, "cost": 0.014606052500000001, "time": 33.792596000432965}, "bfa93a259c": {"quality": 0.25311230436230436, "cost": 0.02601113875, "time": 29.902825361490248}, "c127942d8c": {"quality": 0.2601658757908758, "cost": 0.0032753834999999995, "time": 25.132321882247922}, "c14216d744": {"quality": 0.23213078588078587, "cost": 0.01049051375, "time": 26.48539803624153}, "c2f0bc6921": {"quality": 0.15319229381729382, "cost": 0.0006038535000000001, "time": 16.494534188508986}, "c440b67f31": {"quality": 0.19924325674325674, "cost": 0.030812481500000002, "time": 25.34757208228111}, "caa29fbe8c": {"quality": 0.229778971028971, "cost": 0.025535755, "time": 26.289914256334306}, "cd46331132": {"quality": 0.2289372433122433, "cost": 0.0059953365, "time": 16.957564985752107}, "cdd01242f5": {"quality": 0.23936230436230438, "cost": 0.030000593500000002, "time": 23.897997051477432}, "cdf27f19d3": {"quality": 0.21152472527472527, "cost": 0.00597143625, "time": 21.69409868121147}, "ce7d236454": {"quality": 0.21988629426129425, "cost": 0.0319162425, "time": 34.63264524936676}, "d117c8e23b": {"quality": 0.24436230436230436, "cost": 0.04203393050000001, "time": 37.38888673782348}, "d27a5ad38a": {"quality": 0.19329240204240206, "cost": 0.025790743000000005, "time": 19.247011250257493}, "d287ceaae7": {"quality": 0.26063124375624375, "cost": 0.036845655000000005, "time": 33.75713464021683}, "d29027747a": {"quality": 0.12741758241758241, "cost": 0.0004609135, "time": 47.16611239314079}, "d2d7780d31": {"quality": 0.2296599234099234, "cost": 0.033220648000000005, "time": 29.299119901657104}, "d566a94f3a": {"quality": 0.2568127011877012, "cost": 0.023108875, "time": 7.455539703369141}, "d64dd29abb": {"quality": 0.2582205294705294, "cost": 0.039288317, "time": 35.80952010750771}, "d75495aed3": {"quality": 0.24019563769563768, "cost": 0.0183352845, "time": 36.634383380413055}, "d93337a87d": {"quality": 0.24102897102897103, "cost": 0.010006869500000001, "time": 21.931237626075745}, "de08b84b51": {"quality": 0.23914557664557662, "cost": 0.005503559999999999, "time": 15.377897375822068}, "de39f4f113": {"quality": 0.19986825674325676, "cost": 0.047524011, "time": 18.598653775453567}, "de83615ac5": {"quality": 0.27044275169275167, "cost": 0.00756211075, "time": 28.75660304427147}, "dfa1e5267d": {"quality": 0.2558807858807859, "cost": 0.0013494974999999998, "time": 9.088204801082611}, "dfedfba6df": {"quality": 0.2490950715950716, "cost": 0.004502494499999999, "time": 26.490580713748933}, "e15d59fc9d": {"quality": 0.2783206376956377, "cost": 0.011371385500000001, "time": 52.64762927293778}, "e540f66a9d": {"quality": 0.1645820845820846, "cost": 0.025751229500000007, "time": 19.154958713054658}, "e5c7c9fce2": {"quality": 0.24218129093129093, "cost": 0.0258564275, "time": 28.940241599082945}, "e73beb7df9": {"quality": 0.1834839466089466, "cost": 0.02164974225, "time": 23.38083545565605}, "e939aaa8b3": {"quality": 0.21797341547341548, "cost": 0.028107724500000004, "time": 34.55525960922241}, "e9e55ba5db": {"quality": 0.1835701798201798, "cost": 0.02538775525, "time": 25.480451303720475}, "ea817a074f": {"quality": 0.1976956376956377, "cost": 0.031266095, "time": 24.096565490961076}, "eca3d11e40": {"quality": 0.1920706376956377, "cost": 0.0259032935, "time": 21.720754396915435}, "ed945b7e4a": {"quality": 0.21056270118770118, "cost": 0.03181901100000001, "time": 23.55217906832695}, "ee4ddebe70": {"quality": 0.21718129093129093, "cost": 0.007574288250000001, "time": 33.07409880757332}, "ef62e4b003": {"quality": 0.1578144078144078, "cost": 0.0220901955, "time": 19.725072503089905}, "efe8e44ba4": {"quality": 0.25713078588078586, "cost": 0.00423088875, "time": 25.126209509372714}, "efebd8ca8a": {"quality": 0.12043345543345543, "cost": 0.006324273500000001, "time": 43.206120282411575}, "f0a1a27ee9": {"quality": 0.25917291042291035, "cost": 0.009095101500000001, "time": 33.561097860336304}, "f1c116068c": {"quality": 0.23332244144744144, "cost": 0.00174076875, "time": 18.031138664484025}, "f2c938846b": {"quality": 0.262278971028971, "cost": 0.025320526250000003, "time": 19.31320353746414}, "f462950863": {"quality": 0.14478951603951604, "cost": 0.0007027954999999999, "time": 15.553800004720689}, "f53c8d7a08": {"quality": 0.23630674880674882, "cost": 0.0337485395, "time": 42.32889723777771}, "f570083655": {"quality": 0.21157523032523032, "cost": 0.014973397000000003, "time": 23.602320194244385}, "f5f303dab2": {"quality": 0.2371788628038628, "cost": 0.032701334, "time": 22.13268209695816}, "f70f78f75a": {"quality": 0.27127698690198687, "cost": 0.016446131000000003, "time": 12.967699444293977}, "f767c00a27": {"quality": 0.18858911921411922, "cost": 0.007980581, "time": 24.888762706518172}, "f9cf3c3d99": {"quality": 0.23966144966144964, "cost": 0.014676935000000002, "time": 24.44706709384918}, "f9e84d96b6": {"quality": 0.234640984015984, "cost": 0.0256412165, "time": 28.137858784198762}, "faa5cc0998": {"quality": 0.23488719613719616, "cost": 0.0153379235, "time": 29.529147928953172}, "faa744b8b7": {"quality": 0.1879650210900211, "cost": 0.008215464999999998, "time": 26.75429188609123}, "fabf1b0f34": {"quality": 0.2536892274392274, "cost": 0.0017171129999999997, "time": 13.97360804080963}, "fb53f9a5e6": {"quality": 0.22727897102897102, "cost": 0.031424654, "time": 43.32508450746536}, "fe3b87fd92": {"quality": 0.2024367993117993, "cost": 0.003581219, "time": 22.43371703028679}} ================================================ FILE: abacus-research/biodex-revision-priors-mincost.json ================================================ {"00b02360ef": {"quality": 0.25047369297369293, "cost": 0.038505449000000004, "time": 33.970101940631864}, "025a41642e": {"quality": 0.2696782384282384, "cost": 0.010557185, "time": 31.072653269767763}, "03c4bc9bb0": {"quality": 0.2581852869352869, "cost": 0.010541456000000001, "time": 16.739641118049622}, "04ed954ae9": {"quality": 0.2958458208458209, "cost": 0.046920197000000004, "time": 19.630667555332185}, "059dee8af9": {"quality": 0.23122460872460873, "cost": 0.008803890499999998, "time": 29.826552951335906}, "05fde83b5e": {"quality": 0.20872460872460877, "cost": 0.006782864000000001, "time": 30.33404096364975}, "06b62ff472": {"quality": 0.2968228993228993, "cost": 0.005058610000000001, "time": 14.343733203411102}, "0730a1221a": {"quality": 0.26417915417915416, "cost": 0.026461371, "time": 30.643629491329193}, "07aec969dd": {"quality": 0.22977744477744477, "cost": 0.000723774, "time": 38.15098943710327}, "0a11f0ae2e": {"quality": 0.26241633366633366, "cost": 0.009693431000000002, "time": 17.0743106007576}, "0aeb79b24c": {"quality": 0.19304639804639806, "cost": 0.0058567, "time": 8.773907899856567}, "0b3eafcb23": {"quality": 0.16931984681984683, "cost": 0.0012721839999999998, "time": 16.44918472766876}, "0bc5bbe9f8": {"quality": 0.2640023865023865, "cost": 0.030501102000000002, "time": 42.8920240521431}, "0c328ddadf": {"quality": 0.21182165057165053, "cost": 0.007213844999999999, "time": 25.052031695842743}, "0db4d3e16d": {"quality": 0.24025238650238653, "cost": 0.003130782000000001, "time": 18.73849000930786}, "0df97ab2f2": {"quality": 0.2226345876345876, "cost": 0.0073693990000000004, "time": 38.507711839675906}, "0e0162da5c": {"quality": 0.2489013764013764, "cost": 0.026232925000000008, "time": 28.133001935482024}, "0e0216352f": {"quality": 0.17297064047064048, "cost": 0.008939957, "time": 34.948954832553866}, "0ecd21b93a": {"quality": 0.20069014319014317, "cost": 0.025977679000000004, "time": 28.50129954814911}, "0ff2bde030": {"quality": 0.2610857198357198, "cost": 0.031667818, "time": 16.528453254699706}, "1059f21a6c": {"quality": 0.2190234765234765, "cost": 0.000557161, "time": 9.67062486410141}, "112b665311": {"quality": 0.2551910589410589, "cost": 0.004582068, "time": 23.5158061504364}, "11db1dbc4f": {"quality": 0.23640137640137643, "cost": 0.003211713, "time": 21.993061244487762}, "13145c10ec": {"quality": 0.3172365134865135, "cost": 0.030289498000000005, "time": 31.429670691490173}, "147915874a": {"quality": 0.19364468864468862, "cost": 0.00046279649999999997, "time": 40.85908321142197}, "16e1e2aca6": {"quality": 0.219499666999667, "cost": 0.005164350000000001, "time": 12.553999137878417}, "1914dfad94": {"quality": 0.24569014319014312, "cost": 0.024254144999999998, "time": 19.75359899997711}, "1ab9dcd7d6": {"quality": 0.29846792096792096, "cost": 0.001003366, "time": 21.74862095117569}, "1dc749411d": {"quality": 0.20150238650238647, "cost": 0.009444109000000001, "time": 15.746719026565552}, "1dd84b0308": {"quality": 0.2539410589410589, "cost": 0.026399116, "time": 27.934893941879274}, "1dd9d9bef2": {"quality": 0.2544617882117882, "cost": 0.007334834500000001, "time": 31.84605222940445}, "1f0ba92b8b": {"quality": 0.27228105228105226, "cost": 0.016661861, "time": 33.20399481058121}, "1f1b5a019d": {"quality": 0.2561663336663337, "cost": 0.03240448, "time": 22.484670639038086}, "1f41a35dd3": {"quality": 0.07833333333333332, "cost": 0.004150375499999999, "time": 15.255917310714722}, "212c3fccbf": {"quality": 0.3162624875124875, "cost": 0.030542124999999996, "time": 23.602639937400816}, "219d51cd03": {"quality": 0.1531672494172494, "cost": 0.0008238629999999999, "time": 15.20914809703827}, "238ed27a56": {"quality": 0.28265318015318014, "cost": 0.028584483000000004, "time": 29.401537466049195}, "23f3d97d47": {"quality": 0.28292915417915415, "cost": 0.026455834000000004, "time": 34.8084576010704}, "253547264d": {"quality": 0.20348651348651345, "cost": 0.0261750685, "time": 20.272440803050994}, "27e14a0973": {"quality": 0.2700124875124875, "cost": 0.031140450000000004, "time": 27.74119987487793}, "29960739d3": {"quality": 0.18854728604728602, "cost": 0.030525630000000005, "time": 26.206378316879274}, "2b0a8d5614": {"quality": 0.2262060162060162, "cost": 0.021956766, "time": 16.901934444904327}, "2ccd104a68": {"quality": 0.23872460872460874, "cost": 0.017867876999999997, "time": 46.65757336616516}, "2d1b0cd5d9": {"quality": 0.2476134976134976, "cost": 0.005795422000000001, "time": 17.555127108097075}, "2dcffa3d30": {"quality": 0.3144208569208569, "cost": 0.008584119999999999, "time": 24.954598808288573}, "2ebe033e6a": {"quality": 0.2554706404706405, "cost": 0.007171497000000001, "time": 16.872591602802277}, "31e8ef9410": {"quality": 0.06845238095238096, "cost": 0.0007121749999999999, "time": 75.84835036993027}, "32a628c971": {"quality": 0.31888333888333886, "cost": 0.035293712000000005, "time": 27.239106237888336}, "336e1a4fdc": {"quality": 0.22028138528138527, "cost": 0.030691956000000003, "time": 36.1869512796402}, "33e0fdf4e1": {"quality": 0.2365234765234765, "cost": 0.005039102, "time": 23.91675148010254}, "38a60c768a": {"quality": 0.2449529637029637, "cost": 0.025963075000000002, "time": 34.96920952796936}, "39bb54fbfb": {"quality": 0.29789127539127536, "cost": 0.030766864999999997, "time": 45.52991482019424}, "3ace395562": {"quality": 0.24872460872460872, "cost": 0.024912788, "time": 23.518352210521698}, "3bbe08bf63": {"quality": 0.33796703296703295, "cost": 0.032394053000000006, "time": 37.81355751752854}, "3caf8d77c5": {"quality": 0.3083079420579421, "cost": 0.03448324800000001, "time": 29.443913543224333}, "3d25abc0cb": {"quality": 0.2672743922743922, "cost": 0.008724563000000001, "time": 37.292464637756346}, "3e19290ef5": {"quality": 0.29275238650238655, "cost": 0.024963852000000005, "time": 22.140427541732787}, "3e8e089a02": {"quality": 0.280807942057942, "cost": 0.0233879175, "time": 11.965235376358033}, "3f1771caee": {"quality": 0.143228576978577, "cost": 0.003751839, "time": 24.353824079036713}, "3f7beb53c4": {"quality": 0.29019411144411145, "cost": 0.00168912, "time": 15.684711122512818}, "4081f36f1d": {"quality": 0.2203912753912754, "cost": 0.008877949000000001, "time": 35.81259698867798}, "410edf5f69": {"quality": 0.2731690531690532, "cost": 0.032445931, "time": 40.782276201248166}, "44ec78f301": {"quality": 0.2575774225774226, "cost": 0.030435683499999998, "time": 29.96491810083389}, "44eeb11408": {"quality": 0.27455794205794204, "cost": 0.026113869, "time": 22.3088561296463}, "45623e6def": {"quality": 0.34303016428016425, "cost": 0.032769938, "time": 30.375658011436464}, "470749179f": {"quality": 0.26199966699966704, "cost": 0.004939885999999999, "time": 24.02986795902252}, "47ea56eba0": {"quality": 0.2065848040848041, "cost": 0.017984692000000004, "time": 29.315977609157564}, "49fc8b3768": {"quality": 0.16132201132201132, "cost": 0.002801354, "time": 18.934259843826293}, "4a60adbf47": {"quality": 0.17342823842823843, "cost": 0.013229327499999999, "time": 26.285319995880126}, "4ba372d292": {"quality": 0.26265623265623267, "cost": 0.0073457165, "time": 27.0092391371727}, "4bf38b1922": {"quality": 0.24983571983571987, "cost": 0.030574272000000006, "time": 43.60012021064758}, "4c415b4b86": {"quality": 0.15533175158175158, "cost": 0.024658497000000005, "time": 14.953267228603362}, "4d1fc31b49": {"quality": 0.3281690531690532, "cost": 0.023400054000000003, "time": 16.318808138370514}, "4d8c84f53a": {"quality": 0.2701134976134976, "cost": 0.026254961000000004, "time": 29.027335906028746}, "4df2245b9b": {"quality": 0.12687756687756688, "cost": 0.001181844, "time": 4.539927685260773}, "4e9274d39b": {"quality": 0.2406522644022644, "cost": 0.005825131000000001, "time": 15.826024615764618}, "4f87ed7c2e": {"quality": 0.2662246087246087, "cost": 0.008736922999999999, "time": 27.13904390335083}, "51a340bebc": {"quality": 0.24584582084582088, "cost": 0.0313118765, "time": 28.96012043952942}, "52dee74e25": {"quality": 0.1881456043956044, "cost": 0.031453754, "time": 34.28346945047379}, "533fddb3bd": {"quality": 0.13842823842823843, "cost": 0.00165954, "time": 9.557414150238037}, "5358cb5855": {"quality": 0.18078934953934955, "cost": 0.026390755999999998, "time": 21.654051101207735}, "53b6c3e00f": {"quality": 0.25569014319014316, "cost": 0.000717601, "time": 10.53179224729538}, "5400c14640": {"quality": 0.30376248751248747, "cost": 0.028650208999999992, "time": 38.553849005699156}, "55e373a7e2": {"quality": 0.12201770451770451, "cost": 0.039114879000000005, "time": 18.304805862903596}, "5652070ecb": {"quality": 0.21717698967698967, "cost": 0.007322918, "time": 32.560013604164126}, "56c5cbc75a": {"quality": 0.21707903207903206, "cost": 0.000547569, "time": 38.314506363868716}, "577b168d21": {"quality": 0.22071178821178825, "cost": 0.006765484, "time": 12.956210255622864}, "57cabf0e4b": {"quality": 0.21348651348651346, "cost": 0.024673236, "time": 20.83515272140503}, "58dbfc0499": {"quality": 0.3025124875124875, "cost": 0.053043769000000004, "time": 33.01721404790878}, "5a5739c3d2": {"quality": 0.16318528693528694, "cost": 0.006033024, "time": 30.727768671512603}, "5b12979e52": {"quality": 0.15083694083694082, "cost": 0.0036988935, "time": 18.811239528656007}, "5b384da665": {"quality": 0.24699841824841826, "cost": 0.014450417999999998, "time": 25.65202077627182}, "5b837baf2e": {"quality": 0.2600124875124875, "cost": 0.03879646400000001, "time": 38.47186998128891}, "5c8acd9a75": {"quality": 0.2188238150738151, "cost": 0.026283571999999998, "time": 32.48901460170746}, "5da68874af": {"quality": 0.14364649239649238, "cost": 0.008979280499999999, "time": 33.84627612829208}, "5e304846b6": {"quality": 0.11458083583083582, "cost": 0.007788067, "time": 22.096365022659302}, "5e608d2a8b": {"quality": 0.15110375735375733, "cost": 0.0008234890000000002, "time": 19.88741739988327}, "5e6ad9fb8a": {"quality": 0.24834582084582085, "cost": 0.024505677000000003, "time": 15.4741956949234}, "5ec5b185c9": {"quality": 0.2114807414807415, "cost": 0.0023043549999999997, "time": 37.4191596031189}, "608e332141": {"quality": 0.34233571983571986, "cost": 0.035189383000000005, "time": 22.62248021364212}, "610753ffe5": {"quality": 0.31424048174048175, "cost": 0.029135328, "time": 19.57549432516098}, "614e59c7e7": {"quality": 0.2492218892218892, "cost": 0.001694424, "time": 18.55128355026245}, "62320710f0": {"quality": 0.3199746087246087, "cost": 0.03502582600000001, "time": 31.618607234954833}, "649df27d73": {"quality": 0.2782466144966145, "cost": 0.032545105, "time": 19.415873074531554}, "64f956ba72": {"quality": 0.2931690531690532, "cost": 0.026772999999999998, "time": 9.635773408412934}, "65657a3d04": {"quality": 0.20616633366633366, "cost": 0.011125794, "time": 52.96317781209946}, "6604abbd43": {"quality": 0.30366633366633367, "cost": 0.025960788500000005, "time": 24.2287339925766}, "67050fa89b": {"quality": 0.25265623265623266, "cost": 0.030506207, "time": 39.747187566757205}, "67f05c5985": {"quality": 0.24658300033300035, "cost": 0.02588163, "time": 20.159594106674195}, "67fe27bce0": {"quality": 0.3328912753912754, "cost": 0.039865689, "time": 32.06505537033081}, "698d1b1d0e": {"quality": 0.31830794205794205, "cost": 0.03946355800000001, "time": 51.54699250459671}, "69e8aa9f73": {"quality": 0.18985680985680986, "cost": 0.007278381, "time": 38.63639385700226}, "6b240a8971": {"quality": 0.20597027972027973, "cost": 0.008727093000000002, "time": 28.28091263771057}, "6b56430005": {"quality": 0.3260460372960373, "cost": 0.026531200000000005, "time": 41.742786502838136}, "6b611b7193": {"quality": 0.2922743922743923, "cost": 0.066505375, "time": 17.20961241722107}, "6b77ef93d3": {"quality": 0.17878232878232875, "cost": 0.001498626, "time": 14.678290736675262}, "6beefce17b": {"quality": 0.30852439227439227, "cost": 0.030691372000000005, "time": 24.38176097869873}, "6cb7c9a802": {"quality": 0.33327006327006325, "cost": 0.030604636500000004, "time": 39.88295543193817}, "6cba1da5b3": {"quality": 0.15214035964035963, "cost": 0.013090929000000001, "time": 21.816087329387663}, "6dea54bf99": {"quality": 0.2879309579309579, "cost": 0.004796466000000001, "time": 23.340252709388732}, "6eda6f7780": {"quality": 0.2093607781107781, "cost": 0.010549649000000001, "time": 36.39340627193451}, "708f3ca5da": {"quality": 0.13352439227439225, "cost": 0.0014636204999999998, "time": 14.394513404369354}, "72d75a5f01": {"quality": 0.22862359862359863, "cost": 0.014409670999999999, "time": 24.5260990858078}, "72d99dee93": {"quality": 0.2751513764013764, "cost": 0.028088187000000008, "time": 41.42688899040222}, "73272e5bd3": {"quality": 0.3387246087246087, "cost": 0.030827000000000004, "time": 27.08615951538086}, "74608570f5": {"quality": 0.11727564102564103, "cost": 0.000531192, "time": 25.665570771694185}, "7531840182": {"quality": 0.1608241758241758, "cost": 0.0018236030000000005, "time": 65.24554077386856}, "77ba656673": {"quality": 0.15628538128538128, "cost": 0.008854519500000001, "time": 28.12923115491867}, "77fe41f02f": {"quality": 0.15589035964035963, "cost": 0.042701628000000005, "time": 38.7138086438179}, "7857e95756": {"quality": 0.20908480408480407, "cost": 0.014196858000000001, "time": 26.021716463565827}, "78984e94f0": {"quality": 0.19880702630702632, "cost": 0.0067921480000000005, "time": 24.860538172721864}, "7986633543": {"quality": 0.25714035964035964, "cost": 0.0073338084999999996, "time": 35.33250515460968}, "7bdbc32b57": {"quality": 0.3116196303696303, "cost": 0.0451492925, "time": 26.672986245155336}, "7c89cff787": {"quality": 0.1501694139194139, "cost": 0.0016053209999999998, "time": 20.911856007575988}, "7dc51ec191": {"quality": 0.34112359862359865, "cost": 0.039412432000000004, "time": 45.49512637853623}, "7e4a6245ac": {"quality": 0.13822802197802198, "cost": 0.0004819685, "time": 76.24876043796539}, "7ebc55ebcf": {"quality": 0.3203912753912754, "cost": 0.008825814999999997, "time": 52.15610808134079}, "7f14114a48": {"quality": 0.21598651348651346, "cost": 0.012735542499999999, "time": 17.204772758483887}, "7fb587503e": {"quality": 0.22814560439560436, "cost": 0.009006149000000001, "time": 18.705357003211976}, "7ff82b6f8d": {"quality": 0.22926948051948054, "cost": 0.028896988000000002, "time": 20.79049743413925}, "806a5ef096": {"quality": 0.27122460872460874, "cost": 0.023206115, "time": 11.138013732433318}, "8156d78e42": {"quality": 0.251999666999667, "cost": 0.0073474729999999985, "time": 36.957821094989775}, "815cfb848c": {"quality": 0.2520958208458209, "cost": 0.0247796475, "time": 23.402184796333312}, "820b42e0b1": {"quality": 0.2400949050949051, "cost": 0.002655385, "time": 22.863080990314483}, "82a733626e": {"quality": 0.30914127539127534, "cost": 0.028937054, "time": 23.70889393091202}, "83d6300ced": {"quality": 0.2575124875124875, "cost": 0.008893523, "time": 39.3106644153595}, "844c822d56": {"quality": 0.15672369297369299, "cost": 0.02642007, "time": 22.442726397514342}, "84f9f20f75": {"quality": 0.2695976245976246, "cost": 0.023199555000000004, "time": 15.404870545864105}, "8525986a00": {"quality": 0.2975124875124875, "cost": 0.030195346000000005, "time": 38.38539198637009}, "85db012139": {"quality": 0.2975142912642913, "cost": 0.04670653, "time": 23.935909843444826}, "85ddc8a71e": {"quality": 0.2362246087246087, "cost": 0.005867745, "time": 22.264061617851258}, "8767267bed": {"quality": 0.1742567155067155, "cost": 0.006601124, "time": 7.172838580608368}, "882238f677": {"quality": 0.14891941391941393, "cost": 0.003255827, "time": 27.542255055904388}, "88f78bdeaf": {"quality": 0.09306623931623932, "cost": 0.000593519, "time": 27.451544547080992}, "8af1605300": {"quality": 0.29047369297369297, "cost": 0.007263364000000003, "time": 27.19462056159973}, "8c09cba5e2": {"quality": 0.24566905316905316, "cost": 0.0249850575, "time": 30.331667792797088}, "8e5d966cf6": {"quality": 0.13822802197802198, "cost": 0.001409651, "time": 68.11073198318482}, "8eacbc7240": {"quality": 0.21547369297369298, "cost": 0.012992563000000002, "time": 23.104503071308137}, "8fc917c575": {"quality": 0.24955794205794207, "cost": 0.007428123, "time": 26.517943835258485}, "8fe43ec148": {"quality": 0.25275724275724276, "cost": 0.0030687225, "time": 27.1856760263443}, "900de152da": {"quality": 0.28334582084582083, "cost": 0.026874375, "time": 8.219515359401703}, "901ab65b55": {"quality": 0.34566905316905316, "cost": 0.027035125000000004, "time": 9.139500224590302}, "918182aa18": {"quality": 0.18560592185592187, "cost": 0.002369203, "time": 56.10349173545838}, "91ac37a21a": {"quality": 0.128228021978022, "cost": 0.006092536000000001, "time": 74.77904181480407}, "9252afa4ec": {"quality": 0.2821373071373071, "cost": 0.024976994000000002, "time": 19.884121739864348}, "939b0c3ed9": {"quality": 0.24926157176157174, "cost": 0.009163022, "time": 20.210047280788423}, "93a705cb0d": {"quality": 0.25301157176157174, "cost": 0.01034815, "time": 28.735098361968994}, "94091e2968": {"quality": 0.20182109557109557, "cost": 0.010538336, "time": 30.044746005535124}, "945a9565fa": {"quality": 0.177733793983794, "cost": 0.005453404, "time": 10.252689445018769}, "946baa620a": {"quality": 0.2701134976134976, "cost": 0.024427862, "time": 11.070768821239472}, "950ed976dd": {"quality": 0.09488455988455988, "cost": 0.0034412205, "time": 20.066544604301452}, "955689ee8f": {"quality": 0.17957417582417584, "cost": 0.0017661469999999998, "time": 37.72812922000885}, "95c9b1b1dd": {"quality": 0.2563003663003663, "cost": 0.030597902, "time": 27.93373385667801}, "95ebba630d": {"quality": 0.19334887334887335, "cost": 0.004341687, "time": 18.660895478725433}, "966282f059": {"quality": 0.16358821733821732, "cost": 0.022551696000000003, "time": 21.573746180534364}, "96c401772b": {"quality": 0.2422365134865135, "cost": 0.0026645590000000004, "time": 17.938199400901794}, "9718d880f6": {"quality": 0.20941905316905318, "cost": 0.030509630000000003, "time": 42.891600167751314}, "971ae914cb": {"quality": 0.3079922854922855, "cost": 0.024098346, "time": 11.75359138250351}, "97566e70dd": {"quality": 0.21477439227439227, "cost": 0.016637812999999998, "time": 31.738031923770905}, "97c882ca1f": {"quality": 0.2701134976134976, "cost": 0.034874281, "time": 22.407317924499512}, "98e28fbb93": {"quality": 0.18297369297369298, "cost": 0.009208929000000001, "time": 11.046243464946746}, "992ab25916": {"quality": 0.24106893106893104, "cost": 0.001710615, "time": 18.1801389336586}, "9a271dff29": {"quality": 0.20304639804639804, "cost": 0.009020588999999999, "time": 25.38304090499878}, "9aa6fd8b03": {"quality": 0.2443198468198468, "cost": 0.0058449355000000005, "time": 14.622885823249817}, "9b12b7d441": {"quality": 0.3142615717615717, "cost": 0.010453449, "time": 24.59365568161011}, "9bf31a2127": {"quality": 0.2737246087246087, "cost": 0.032263425, "time": 41.504128229618075}, "9c4ad5aed3": {"quality": 0.2762624875124875, "cost": 0.024755695000000005, "time": 21.714357328414916}, "9cbb34ee5f": {"quality": 0.23064740814740814, "cost": 0.0023012260000000004, "time": 23.29413582086563}, "9e16d7ba9c": {"quality": 0.25935772560772563, "cost": 0.031369863000000005, "time": 33.015732765197754}, "9e1a7dd196": {"quality": 0.19444194694194694, "cost": 0.0035900009999999998, "time": 15.451323175430298}, "9e6775ac33": {"quality": 0.06856227106227106, "cost": 0.0017025660000000002, "time": 15.161614441871643}, "9e8674b6c7": {"quality": 0.25872460872460873, "cost": 0.032513841, "time": 38.995942330360414}, "9ea12d8d80": {"quality": 0.14647893772893772, "cost": 0.006858684000000001, "time": 12.53976230621338}, "a232f1dfb6": {"quality": 0.2482117882117882, "cost": 0.0017046329999999999, "time": 17.978844976425172}, "a3b7cb6c33": {"quality": 0.24213369963369963, "cost": 0.00900874, "time": 19.766272163391115}, "a72f24f687": {"quality": 0.21526862026862026, "cost": 0.007625456000000003, "time": 14.816489315032959}, "a8aac20fd9": {"quality": 0.33620115995115996, "cost": 0.027009404000000004, "time": 20.422273874282837}, "a9e811d4a7": {"quality": 0.27455794205794204, "cost": 0.023571902, "time": 17.510642659664153}, "aa1fc47a86": {"quality": 0.23011349761349761, "cost": 0.024848436999999998, "time": 23.83184322118759}, "aa67b102e7": {"quality": 0.29247460872460873, "cost": 0.029215488, "time": 25.6765993475914}, "ad453a813f": {"quality": 0.2581474081474081, "cost": 0.0031871219999999997, "time": 17.01520048379898}, "add25d67a0": {"quality": 0.19999750249750248, "cost": 0.027308536, "time": 21.267851161956788}, "adf6ae1ba7": {"quality": 0.15185772560772562, "cost": 0.0030511870000000003, "time": 10.442331075668335}, "af57afe626": {"quality": 0.25730269730269734, "cost": 0.0027442350000000003, "time": 20.559476172924043}, "b07ae35700": {"quality": 0.25820693195693195, "cost": 0.0031182555, "time": 21.726053476333618}, "b09741c8f7": {"quality": 0.13642246642246642, "cost": 0.00039643100000000004, "time": 36.347797584533694}, "b1b81c0847": {"quality": 0.35997460872460874, "cost": 0.0234363775, "time": 12.63291654586792}, "b38fbeda99": {"quality": 0.18547369297369298, "cost": 0.008016031, "time": 29.251738607883453}, "b3b92b1835": {"quality": 0.24926157176157174, "cost": 0.04590919, "time": 22.62844548225403}, "b3bba3eee2": {"quality": 0.19231532356532358, "cost": 0.007909536, "time": 42.60630009174347}, "b40727bd68": {"quality": 0.2634468309468309, "cost": 0.024455202000000006, "time": 20.662753903865813}, "b6a0f78896": {"quality": 0.264456931956932, "cost": 0.008735882999999998, "time": 34.55237965583801}, "b84e122880": {"quality": 0.2734468309468309, "cost": 0.008659572, "time": 40.34617894887924}, "b8c6c44a39": {"quality": 0.30366633366633367, "cost": 0.0289365195, "time": 22.390970599651336}, "bb9ed9dffe": {"quality": 0.16283660783660783, "cost": 0.001536843, "time": 17.358729672431945}, "bbc50a0411": {"quality": 0.22165931290931287, "cost": 0.0031182930000000003, "time": 28.581140315532686}, "bbd0498616": {"quality": 0.22491633366633365, "cost": 0.051756142000000005, "time": 17.691377997398376}, "bd018ec9d7": {"quality": 0.19921703296703297, "cost": 0.0005695385, "time": 72.71849246025086}, "be11a8a86e": {"quality": 0.2582117882117882, "cost": 0.013947379000000001, "time": 26.179900193214415}, "bfa93a259c": {"quality": 0.3037246087246087, "cost": 0.0248712575, "time": 27.526713395118712}, "c127942d8c": {"quality": 0.2636650849150849, "cost": 0.0032525069999999995, "time": 24.242635214328764}, "c14216d744": {"quality": 0.24926157176157174, "cost": 0.010124539499999998, "time": 25.00144135951996}, "c2f0bc6921": {"quality": 0.14888458763458762, "cost": 0.000605392, "time": 14.55265941619873}, "c440b67f31": {"quality": 0.21598651348651346, "cost": 0.02981187, "time": 24.81428416967392}, "caa29fbe8c": {"quality": 0.27789127539127534, "cost": 0.0247254, "time": 26.996189880371094}, "cd46331132": {"quality": 0.30287448662448657, "cost": 0.005876232, "time": 16.816848158836365}, "cdd01242f5": {"quality": 0.3245579420579421, "cost": 0.029167213999999997, "time": 18.141655015945435}, "cdf27f19d3": {"quality": 0.2780494505494505, "cost": 0.005842002000000001, "time": 15.971444475650788}, "ce7d236454": {"quality": 0.28977258852258847, "cost": 0.030690639, "time": 30.128969633579253}, "d117c8e23b": {"quality": 0.3412246087246087, "cost": 0.03989662200000001, "time": 39.21010599136353}, "d27a5ad38a": {"quality": 0.1940848040848041, "cost": 0.024636320000000003, "time": 19.60495171546936}, "d287ceaae7": {"quality": 0.3262624875124875, "cost": 0.035333185, "time": 32.71745250225067}, "d29027747a": {"quality": 0.0815018315018315, "cost": 0.000420386, "time": 37.273941445350644}, "d2d7780d31": {"quality": 0.2843198468198468, "cost": 0.031180686000000006, "time": 28.442743134498595}, "d566a94f3a": {"quality": 0.30195873570873577, "cost": 0.02176175, "time": 8.928415286540986}, "d64dd29abb": {"quality": 0.3114410589410589, "cost": 0.03748003400000001, "time": 31.35522733926773}, "d75495aed3": {"quality": 0.24872460872460872, "cost": 0.017243897, "time": 37.48555475473404}, "d93337a87d": {"quality": 0.2603912753912754, "cost": 0.009152621000000001, "time": 21.659664070606233}, "de08b84b51": {"quality": 0.2466244866244866, "cost": 0.005220604999999999, "time": 13.204281628131866}, "de39f4f113": {"quality": 0.1780698468198468, "cost": 0.045369664999999997, "time": 15.614853060245514}, "de83615ac5": {"quality": 0.32421883671883667, "cost": 0.0074163139999999985, "time": 25.368940019607543}, "dfa1e5267d": {"quality": 0.24926157176157174, "cost": 0.001261335, "time": 7.206225287914276}, "dfedfba6df": {"quality": 0.24569014319014312, "cost": 0.0042948735, "time": 20.24753314256668}, "e15d59fc9d": {"quality": 0.33080794205794206, "cost": 0.011172813, "time": 53.19109081029892}, "e540f66a9d": {"quality": 0.14333083583083583, "cost": 0.024458602000000006, "time": 16.43147575855255}, "e5c7c9fce2": {"quality": 0.28936258186258185, "cost": 0.024852987500000003, "time": 23.036948883533476}, "e73beb7df9": {"quality": 0.1644678932178932, "cost": 0.0206705125, "time": 19.65618509054184}, "e939aaa8b3": {"quality": 0.24094683094683092, "cost": 0.026475653000000005, "time": 27.854652655124664}, "e9e55ba5db": {"quality": 0.16214035964035964, "cost": 0.024552966000000002, "time": 22.01356840133667}, "ea817a074f": {"quality": 0.2803912753912754, "cost": 0.030159274000000003, "time": 26.384180903434753}, "eca3d11e40": {"quality": 0.25997460872460876, "cost": 0.024926519, "time": 22.995232653617858}, "ed945b7e4a": {"quality": 0.28362540237540235, "cost": 0.030488778000000005, "time": 23.277513599395753}, "ee4ddebe70": {"quality": 0.2726959151959152, "cost": 0.007441470000000001, "time": 33.65448969602585}, "ef62e4b003": {"quality": 0.15812881562881562, "cost": 0.020466967000000003, "time": 19.30031136274338}, "efe8e44ba4": {"quality": 0.2592615717615717, "cost": 0.0040250775, "time": 21.48128048181534}, "efebd8ca8a": {"quality": 0.11086691086691088, "cost": 0.004719911, "time": 28.901436603069307}, "f0a1a27ee9": {"quality": 0.25251248751248745, "cost": 0.008780909, "time": 30.412869799137116}, "f1c116068c": {"quality": 0.2574782162282162, "cost": 0.001708935, "time": 17.96944682598114}, "f2c938846b": {"quality": 0.32955794205794203, "cost": 0.024549196500000002, "time": 19.827334237098693}, "f462950863": {"quality": 0.18207903207903206, "cost": 0.0006345659999999999, "time": 19.180297017097473}, "f53c8d7a08": {"quality": 0.27761349761349763, "cost": 0.032353539, "time": 41.01129903793335}, "f570083655": {"quality": 0.25315046065046065, "cost": 0.014203856, "time": 25.355476438999176}, "f5f303dab2": {"quality": 0.26935772560772564, "cost": 0.031210957999999994, "time": 26.234438502788542}, "f70f78f75a": {"quality": 0.31005397380397376, "cost": 0.015462705999999998, "time": 15.054353976249695}, "f767c00a27": {"quality": 0.1513449050949051, "cost": 0.007778000499999998, "time": 23.562188839912416}, "f9cf3c3d99": {"quality": 0.30015623265623265, "cost": 0.013173157, "time": 23.90029693841934}, "f9e84d96b6": {"quality": 0.2451153013653013, "cost": 0.024730744000000002, "time": 21.731143033504488}, "faa5cc0998": {"quality": 0.27644105894105897, "cost": 0.014485047000000001, "time": 30.353227543830872}, "faa744b8b7": {"quality": 0.17509670884670886, "cost": 0.007972195999999999, "time": 26.635566544532775}, "fabf1b0f34": {"quality": 0.3082117882117882, "cost": 0.0016653059999999996, "time": 13.257695925235748}, "fb53f9a5e6": {"quality": 0.312057942057942, "cost": 0.030517666500000006, "time": 38.16447761058807}, "fe3b87fd92": {"quality": 0.2373735986235986, "cost": 0.003384599, "time": 24.418578135967255}} ================================================ FILE: abacus-research/cheap-priors-cascades.json ================================================ {"0005c18b69": {"quality": 0.5422666666666667, "cost": 1.62e-06, "time": 0.026}, "009df798a3": {"quality": 0.470225, "cost": 1.59e-06, "time": 0.027999999999999997}, "00c93aec22": {"quality": 0.426725, "cost": 3.1199999999999998e-06, "time": 0.030900000000000004}, "00e1fecc4c": {"quality": 0.48889999999999995, "cost": 2.24e-06, "time": 0.026199999999999998}, "00f4acd0d3": {"quality": 0.49876666666666675, "cost": 3.15e-06, "time": 0.0322}, "01413aa72d": {"quality": 0.44705, "cost": 2.3600000000000003e-06, "time": 0.0293}, "01c2f973ad": {"quality": 0.5318, "cost": 1.95e-06, "time": 0.020999999999999998}, "02078988c1": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.031}, "021604dec1": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "02410c662e": {"quality": 0.4013, "cost": 1.8e-06, "time": 0.0299}, "0262668df7": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "0267c97b70": {"quality": 0.5648333333333334, "cost": 2.7e-06, "time": 0.0308}, "02ae38e4aa": {"quality": 0.45935000000000004, "cost": 1.59e-06, "time": 0.0247}, "02f49fe0fd": {"quality": 0.365, "cost": 2.4e-07, "time": 0.0128}, "030756558c": {"quality": 0.45363333333333333, "cost": 9.9e-07, "time": 0.0226}, "033ca325e6": {"quality": 0.5508500000000001, "cost": 2.12e-06, "time": 0.019799999999999998}, "038a5f0a62": {"quality": 0.5648333333333334, "cost": 2.7e-06, "time": 0.030799999999999998}, "041b5af43d": {"quality": 0.52195, "cost": 3.830000000000001e-06, "time": 0.0413}, "042d933706": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "04397effa0": {"quality": 0.46777500000000005, "cost": 2.43e-06, "time": 0.0333}, "0539e0b42d": {"quality": 0.5050250000000001, "cost": 2.7500000000000004e-06, "time": 0.0365}, "0554568b86": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.0229}, "06493715cc": {"quality": 0.4763, "cost": 1.47e-06, "time": 0.015}, "067ee6e91b": {"quality": 0.49876666666666675, "cost": 3.15e-06, "time": 0.0322}, "068b66f00d": {"quality": 0.4630666666666667, "cost": 2e-06, "time": 0.026699999999999998}, "0695f9b5fc": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "073ed5b301": {"quality": 0.39890000000000003, "cost": 1.6799999999999998e-06, "time": 0.020200000000000003}, "079feb14a8": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.0263}, "07a3a7daf7": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "08127cd6dd": {"quality": 0.6497666666666667, "cost": 4.27e-06, "time": 0.037599999999999995}, "0833133620": {"quality": 0.5536, "cost": 1.86e-06, "time": 0.022199999999999998}, "089565077c": {"quality": 0.365, "cost": 1.2e-07, "time": 0.0064}, "08bf8cc191": {"quality": 0.53045, "cost": 4.07e-06, "time": 0.0375}, "08e1802287": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "090cd3ef31": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "0944a921e8": {"quality": 0.48890000000000006, "cost": 2.24e-06, "time": 0.026199999999999998}, "0947216ece": {"quality": 0.5724666666666667, "cost": 3.88e-06, "time": 0.0304}, "096d51f670": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "09791c731b": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "0990c0d4f8": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "0a128688c1": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.0229}, "0a4c1bbb4a": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.018299999999999997}, "0ac969dde3": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.040799999999999996}, "0af1efab0e": {"quality": 0.4021666666666666, "cost": 8.4e-07, "time": 0.0149}, "0b1ed7ff58": {"quality": 0.5244, "cost": 2.99e-06, "time": 0.036}, "0b3dc2e896": {"quality": 0.45555, "cost": 2.6e-06, "time": 0.025500000000000002}, "0b43e94f3f": {"quality": 0.5121, "cost": 1.88e-06, "time": 0.0203}, "0b4ab72197": {"quality": 0.42146666666666666, "cost": 2.76e-06, "time": 0.025}, "0be862a0dc": {"quality": 0.4762, "cost": 2.07e-06, "time": 0.027399999999999997}, "0bf3129ae8": {"quality": 0.4794666666666667, "cost": 1.23e-06, "time": 0.0221}, "0c020b86a3": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.0224}, "0c6c7fe96a": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "0c81c8996a": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "0cdc5954dd": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.031}, "0d25188bf7": {"quality": 0.4098, "cost": 2.04e-06, "time": 0.0261}, "0d9d767ae5": {"quality": 0.5611333333333334, "cost": 3.6400000000000003e-06, "time": 0.0342}, "0e36342fe7": {"quality": 0.39885000000000004, "cost": 1.3199999999999999e-06, "time": 0.0176}, "0e7e862290": {"quality": 0.4021666666666666, "cost": 8.4e-07, "time": 0.0149}, "0e91cd07f9": {"quality": 0.4762, "cost": 2.07e-06, "time": 0.027399999999999997}, "0ec672e7c8": {"quality": 0.5304500000000001, "cost": 4.07e-06, "time": 0.037500000000000006}, "0ed243f788": {"quality": 0.543775, "cost": 3.23e-06, "time": 0.0355}, "0eeb372802": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.033600000000000005}, "0effe9b1dc": {"quality": 0.40216666666666673, "cost": 8.4e-07, "time": 0.0149}, "0f7faf684d": {"quality": 0.52195, "cost": 3.83e-06, "time": 0.0413}, "0fcec544e3": {"quality": 0.5064500000000001, "cost": 1.9799999999999997e-06, "time": 0.0286}, "0ff126ebf8": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "112d9a3421": {"quality": 0.455475, "cost": 3.2000000000000003e-06, "time": 0.0379}, "114a097c53": {"quality": 0.43270000000000003, "cost": 2.4e-06, "time": 0.0224}, "116334cd72": {"quality": 0.5290250000000001, "cost": 4.84e-06, "time": 0.045399999999999996}, "1175ee37e6": {"quality": 0.5367, "cost": 1.11e-06, "time": 0.0157}, "11a66478dc": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "11bc996d48": {"quality": 0.5648333333333334, "cost": 2.7e-06, "time": 0.030799999999999998}, "11debf9fc0": {"quality": 0.467775, "cost": 2.43e-06, "time": 0.033299999999999996}, "123fb650fb": {"quality": 0.429175, "cost": 2.2799999999999998e-06, "time": 0.0256}, "1274c21076": {"quality": 0.463975, "cost": 3.44e-06, "time": 0.0341}, "127af50739": {"quality": 0.44705, "cost": 2.3600000000000003e-06, "time": 0.0293}, "12addbf5e2": {"quality": 0.3989, "cost": 1.6799999999999998e-06, "time": 0.0202}, "133ee5023f": {"quality": 0.54595, "cost": 2.96e-06, "time": 0.025099999999999997}, "1368e1c78e": {"quality": 0.4021666666666666, "cost": 8.4e-07, "time": 0.0149}, "13a009fe0c": {"quality": 0.5399750000000001, "cost": 4.24e-06, "time": 0.0363}, "13da306f84": {"quality": 0.4969666666666666, "cost": 3.32e-06, "time": 0.0277}, "13f2b9c25b": {"quality": 0.517325, "cost": 1.98e-06, "time": 0.0319}, "13f75f9bd0": {"quality": 0.4392333333333333, "cost": 1.92e-06, "time": 0.023}, "1404e0aa35": {"quality": 0.513525, "cost": 2.99e-06, "time": 0.0327}, "140ededb41": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "142f3a7c70": {"quality": 0.4763, "cost": 1.47e-06, "time": 0.015}, "1468dddecc": {"quality": 0.5318, "cost": 1.95e-06, "time": 0.020999999999999998}, "14d19a01e2": {"quality": 0.4134, "cost": 1.6799999999999998e-06, "time": 0.0235}, "1624bb5302": {"quality": 0.4166666666666667, "cost": 8.4e-07, "time": 0.0182}, "1636e0833b": {"quality": 0.37633333333333335, "cost": 6e-07, "time": 0.0154}, "1658296f3a": {"quality": 0.41340000000000005, "cost": 1.6799999999999998e-06, "time": 0.0235}, "16f351273f": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "171e6ae293": {"quality": 0.5611333333333334, "cost": 3.64e-06, "time": 0.034199999999999994}, "176da24f53": {"quality": 0.5291, "cost": 2.12e-06, "time": 0.0165}, "179379555f": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "17c928174f": {"quality": 0.5422666666666667, "cost": 1.6200000000000002e-06, "time": 0.026}, "181c91d1be": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.030999999999999996}, "18368684cd": {"quality": 0.4134, "cost": 1.6799999999999998e-06, "time": 0.0235}, "183743e76e": {"quality": 0.43596666666666667, "cost": 2.76e-06, "time": 0.0283}, "18f55750b0": {"quality": 0.39892500000000003, "cost": 2.04e-06, "time": 0.0228}, "19563b057d": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.0229}, "1957127275": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.0229}, "197bb53f10": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "199fd1fbf2": {"quality": 0.40375, "cost": 4.8e-07, "time": 0.0123}, "19b40e0271": {"quality": 0.513525, "cost": 2.99e-06, "time": 0.0327}, "19e3db7fe7": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.0336}, "1ad856985f": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "1adec2dca2": {"quality": 0.47247500000000003, "cost": 3.68e-06, "time": 0.0303}, "1b04a2b184": {"quality": 0.5304500000000001, "cost": 4.07e-06, "time": 0.037500000000000006}, "1b28439bd7": {"quality": 0.5724666666666667, "cost": 3.88e-06, "time": 0.0304}, "1beb2fac62": {"quality": 0.4969666666666666, "cost": 3.32e-06, "time": 0.0277}, "1c347e4d91": {"quality": 0.5053, "cost": 1.47e-06, "time": 0.021599999999999998}, "1c3882926e": {"quality": 0.4484, "cost": 2.19e-06, "time": 0.033800000000000004}, "1cc6d9efb6": {"quality": 0.5837, "cost": 4.7200000000000005e-06, "time": 0.03899999999999999}, "1ce3d77039": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "1ce99cf2c8": {"quality": 0.5050250000000001, "cost": 2.7500000000000004e-06, "time": 0.0365}, "1d26090364": {"quality": 0.5318, "cost": 1.95e-06, "time": 0.020999999999999998}, "1d87f97e62": {"quality": 0.6403333333333333, "cost": 3.26e-06, "time": 0.0335}, "1da2369719": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "1e18e60895": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "1e1bf7e88b": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "1e8b3521f8": {"quality": 0.5680999999999999, "cost": 1.86e-06, "time": 0.0255}, "1f5e8c9e9a": {"quality": 0.463975, "cost": 3.44e-06, "time": 0.034100000000000005}, "2018bef45f": {"quality": 0.5147333333333334, "cost": 2.48e-06, "time": 0.025699999999999997}, "2066966577": {"quality": 0.463975, "cost": 3.44e-06, "time": 0.0341}, "2075ff1d04": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "2080b60a57": {"quality": 0.448475, "cost": 1.5899999999999998e-06, "time": 0.021400000000000002}, "208a98f514": {"quality": 0.5800000000000001, "cost": 3.62e-06, "time": 0.03609999999999999}, "20e10af7d4": {"quality": 0.47872499999999996, "cost": 1.8299999999999998e-06, "time": 0.0242}, "20e2c0b057": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "211b89b4cd": {"quality": 0.49876666666666675, "cost": 3.15e-06, "time": 0.0322}, "2153174e2d": {"quality": 0.45792499999999997, "cost": 2.3600000000000003e-06, "time": 0.0326}, "21b2b8ebd1": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "21b2df8512": {"quality": 0.48889999999999995, "cost": 2.24e-06, "time": 0.0262}, "21bed16a7d": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "227246dff8": {"quality": 0.40375, "cost": 4.8e-07, "time": 0.0123}, "227c30d349": {"quality": 0.38766666666666666, "cost": 8.4e-07, "time": 0.011600000000000001}, "228687831a": {"quality": 0.4101333333333333, "cost": 2.52e-06, "time": 0.0288}, "23075b2a6e": {"quality": 0.4649666666666667, "cost": 1.23e-06, "time": 0.018799999999999997}, "23566f15ab": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "2370cebb10": {"quality": 0.45935000000000004, "cost": 1.59e-06, "time": 0.0247}, "2386b03c4c": {"quality": 0.4484, "cost": 2.19e-06, "time": 0.033800000000000004}, "24957f3a43": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.0197}, "24c122de4e": {"quality": 0.4101333333333334, "cost": 2.5199999999999996e-06, "time": 0.0288}, "24f76747b9": {"quality": 0.4649666666666667, "cost": 1.23e-06, "time": 0.018799999999999997}, "2609bfd616": {"quality": 0.4858, "cost": 2.84e-06, "time": 0.0283}, "260ab3e966": {"quality": 0.420675, "cost": 2.04e-06, "time": 0.0294}, "2728c8eb6a": {"quality": 0.42146666666666666, "cost": 2.76e-06, "time": 0.025}, "27971eaaf5": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.0229}, "27ba0964b2": {"quality": 0.40375, "cost": 4.8e-07, "time": 0.0123}, "27daa50458": {"quality": 0.5508500000000001, "cost": 2.12e-06, "time": 0.019799999999999998}, "28369b2421": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.0263}, "28421e6d62": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.022399999999999996}, "2848c42f91": {"quality": 0.4649666666666667, "cost": 1.2299999999999999e-06, "time": 0.0188}, "28a638bb6e": {"quality": 0.45935000000000004, "cost": 1.59e-06, "time": 0.0247}, "290947fe5a": {"quality": 0.45935, "cost": 1.5899999999999998e-06, "time": 0.0247}, "2936c3e43e": {"quality": 0.5304500000000001, "cost": 4.07e-06, "time": 0.0375}, "293ec5edca": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "294e541235": {"quality": 0.51495, "cost": 1.11e-06, "time": 0.0124}, "295ed5e759": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.0197}, "2960431101": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "29892d8468": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.0336}, "299a0aeb65": {"quality": 0.590875, "cost": 3.62e-06, "time": 0.0394}, "29bf3c0a3b": {"quality": 0.45690000000000003, "cost": 2.43e-06, "time": 0.03}, "29c8c693e2": {"quality": 0.52195, "cost": 3.830000000000001e-06, "time": 0.041299999999999996}, "2a7d15f4a7": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "2ae24e0124": {"quality": 0.514875, "cost": 2.82e-06, "time": 0.0372}, "2b5679d248": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "2b5ab72a55": {"quality": 0.365, "cost": 1.2e-07, "time": 0.0064}, "2b82a67eb1": {"quality": 0.45363333333333333, "cost": 9.9e-07, "time": 0.0226}, "2bcbffdf85": {"quality": 0.45935000000000004, "cost": 1.59e-06, "time": 0.0247}, "2bd39ee744": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "2bf38d797f": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "2c1640adf7": {"quality": 0.4794666666666667, "cost": 1.23e-06, "time": 0.0221}, "2c5cf9eb26": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.023599999999999996}, "2c87313a93": {"quality": 0.47240000000000004, "cost": 4.28e-06, "time": 0.0427}, "2c9a9f94c4": {"quality": 0.426725, "cost": 3.1199999999999998e-06, "time": 0.030900000000000004}, "2d3bbc2d23": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "2d7f1dbd4b": {"quality": 0.470225, "cost": 1.59e-06, "time": 0.027999999999999997}, "2de113167b": {"quality": 0.494225, "cost": 3.68e-06, "time": 0.0369}, "2de3eb2c19": {"quality": 0.6592, "cost": 3.52e-06, "time": 0.0278}, "2e02b71061": {"quality": 0.4166666666666667, "cost": 8.4e-07, "time": 0.0182}, "2e30394ac6": {"quality": 0.4135, "cost": 1.08e-06, "time": 0.011099999999999999}, "2e5d071f21": {"quality": 0.45792499999999997, "cost": 2.3600000000000003e-06, "time": 0.0326}, "2e9c5cc9bf": {"quality": 0.5291, "cost": 2.12e-06, "time": 0.0165}, "2ec4bec1a3": {"quality": 0.455475, "cost": 3.2000000000000003e-06, "time": 0.0379}, "2f1573da80": {"quality": 0.429175, "cost": 2.2799999999999998e-06, "time": 0.0256}, "2fc0cb3592": {"quality": 0.43596666666666667, "cost": 2.76e-06, "time": 0.0283}, "2fd9cd426a": {"quality": 0.48335, "cost": 3.68e-06, "time": 0.033600000000000005}, "3019af79b3": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "302c1d97fc": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027699999999999995}, "303b467574": {"quality": 0.5244, "cost": 2.99e-06, "time": 0.036}, "3058b1f1f8": {"quality": 0.4013, "cost": 1.8e-06, "time": 0.0299}, "30ae4cbe91": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "30e3ff1d17": {"quality": 0.47729999999999995, "cost": 2.6e-06, "time": 0.0321}, "30f20c8fe6": {"quality": 0.513525, "cost": 2.99e-06, "time": 0.0327}, "316759d191": {"quality": 0.463975, "cost": 3.44e-06, "time": 0.034100000000000005}, "3177802176": {"quality": 0.365, "cost": 1.2e-07, "time": 0.0064}, "3184f977a8": {"quality": 0.39287500000000003, "cost": 9.6e-07, "time": 0.0213}, "3194e440cf": {"quality": 0.4762, "cost": 2.07e-06, "time": 0.027399999999999997}, "3197ad4faf": {"quality": 0.37633333333333335, "cost": 6e-07, "time": 0.0154}, "31a423a3bf": {"quality": 0.382, "cost": 4.8e-07, "time": 0.009000000000000001}, "321e17afbd": {"quality": 0.46642500000000003, "cost": 2.6e-06, "time": 0.0288}, "32b101d807": {"quality": 0.538875, "cost": 4.9100000000000004e-06, "time": 0.0461}, "332a350ea2": {"quality": 0.4134, "cost": 1.6799999999999998e-06, "time": 0.0235}, "33459cd29c": {"quality": 0.47492500000000004, "cost": 2.84e-06, "time": 0.025}, "33a187e74f": {"quality": 0.525825, "cost": 2.22e-06, "time": 0.0281}, "34026bb5cc": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "3511b5e1d0": {"quality": 0.39899999999999997, "cost": 1.08e-06, "time": 0.0078}, "3513e54767": {"quality": 0.54595, "cost": 2.96e-06, "time": 0.025099999999999997}, "353f0cb1ac": {"quality": 0.5508500000000001, "cost": 2.12e-06, "time": 0.019799999999999998}, "3550bf88cb": {"quality": 0.53425, "cost": 3.06e-06, "time": 0.036699999999999997}, "35610fb420": {"quality": 0.5536, "cost": 1.86e-06, "time": 0.022199999999999998}, "357267e14b": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "36011c7606": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "362d480d6d": {"quality": 0.463975, "cost": 3.44e-06, "time": 0.034100000000000005}, "363209b6e7": {"quality": 0.48890000000000006, "cost": 2.24e-06, "time": 0.026199999999999998}, "3637084f91": {"quality": 0.5147333333333334, "cost": 2.48e-06, "time": 0.025699999999999997}, "36b17c40f3": {"quality": 0.39885000000000004, "cost": 1.3199999999999999e-06, "time": 0.0176}, "36c66671ee": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.023599999999999996}, "372e8b5f4f": {"quality": 0.4134, "cost": 1.6799999999999998e-06, "time": 0.0235}, "375ed248fe": {"quality": 0.5147333333333334, "cost": 2.48e-06, "time": 0.025699999999999997}, "377b8b0bcc": {"quality": 0.45935, "cost": 1.5899999999999998e-06, "time": 0.0247}, "37bd28f2c9": {"quality": 0.5244, "cost": 2.99e-06, "time": 0.036}, "38075bb01f": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "38567d6a43": {"quality": 0.4763, "cost": 1.47e-06, "time": 0.015}, "389a99ab21": {"quality": 0.46532500000000004, "cost": 3.27e-06, "time": 0.0386}, "389c54cbca": {"quality": 0.54595, "cost": 2.96e-06, "time": 0.025099999999999997}, "38ec11cf7b": {"quality": 0.45690000000000003, "cost": 2.43e-06, "time": 0.03}, "3980f20caa": {"quality": 0.48090000000000005, "cost": 4.52e-06, "time": 0.038900000000000004}, "39cd4ca402": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.018299999999999997}, "3a34b24c41": {"quality": 0.4762, "cost": 2.07e-06, "time": 0.0274}, "3b2e8075ea": {"quality": 0.5680999999999999, "cost": 1.86e-06, "time": 0.0255}, "3b3676521a": {"quality": 0.5955, "cost": 5.47e-06, "time": 0.048799999999999996}, "3b3a6bf087": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "3b4bde0121": {"quality": 0.38766666666666666, "cost": 8.4e-07, "time": 0.011600000000000001}, "3b57530a56": {"quality": 0.64505, "cost": 2.51e-06, "time": 0.0237}, "3c206c89f3": {"quality": 0.47485, "cost": 3.44e-06, "time": 0.037399999999999996}, "3cbab8082e": {"quality": 0.523375, "cost": 3.06e-06, "time": 0.0334}, "3d71c4dd2c": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.026899999999999997}, "3ea15ac20c": {"quality": 0.390425, "cost": 1.8e-06, "time": 0.026600000000000002}, "3f1a58aec9": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "3f2321bb08": {"quality": 0.365, "cost": 1.2e-07, "time": 0.0064}, "3f3ef494b0": {"quality": 0.43596666666666667, "cost": 2.76e-06, "time": 0.0283}, "3f62c3fbfc": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "3f88dd99f7": {"quality": 0.5121, "cost": 1.88e-06, "time": 0.0203}, "3fa747af9a": {"quality": 0.3908333333333333, "cost": 6e-07, "time": 0.0187}, "40104c813f": {"quality": 0.522025, "cost": 3.23e-06, "time": 0.028899999999999995}, "403b05da2d": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027699999999999995}, "4043815a3e": {"quality": 0.5206, "cost": 4.000000000000001e-06, "time": 0.0368}, "409ff67607": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.018299999999999997}, "412c065b83": {"quality": 0.4102333333333333, "cost": 1.92e-06, "time": 0.016399999999999998}, "4171fbac5c": {"quality": 0.47485, "cost": 3.44e-06, "time": 0.037399999999999996}, "4191118787": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "41d5b97871": {"quality": 0.43923333333333336, "cost": 1.92e-06, "time": 0.023}, "41d8845655": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "41ee202cac": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "42082dcd0d": {"quality": 0.45690000000000003, "cost": 2.43e-06, "time": 0.030000000000000002}, "42ddd48341": {"quality": 0.537525, "cost": 5.0800000000000005e-06, "time": 0.0416}, "4361bc7ea7": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "43afdad250": {"quality": 0.52195, "cost": 3.83e-06, "time": 0.0413}, "43c3cf9cb8": {"quality": 0.6592, "cost": 3.52e-06, "time": 0.0278}, "43d24fb32a": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027700000000000002}, "440bc872de": {"quality": 0.45690000000000003, "cost": 2.4299999999999996e-06, "time": 0.03}, "44173a9aef": {"quality": 0.5290250000000001, "cost": 4.84e-06, "time": 0.045399999999999996}, "44d6af5523": {"quality": 0.6309, "cost": 2.25e-06, "time": 0.0294}, "44fe4e4e3e": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "4587a1500c": {"quality": 0.4630666666666667, "cost": 2e-06, "time": 0.026699999999999998}, "45ef93b61e": {"quality": 0.4762, "cost": 2.07e-06, "time": 0.0274}, "461846a52d": {"quality": 0.42799999999999994, "cost": 1.08e-06, "time": 0.0144}, "462e6ff849": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.0336}, "4630853d32": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "46475b9e75": {"quality": 0.5536, "cost": 1.86e-06, "time": 0.022199999999999998}, "46654a1f32": {"quality": 0.5837, "cost": 4.7200000000000005e-06, "time": 0.039}, "466a3036b2": {"quality": 0.47382500000000005, "cost": 3.51e-06, "time": 0.0348}, "466d4d16dd": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.022399999999999996}, "46a35022d8": {"quality": 0.49795, "cost": 8.7e-07, "time": 0.0162}, "46ed68152d": {"quality": 0.5002333333333333, "cost": 2.48e-06, "time": 0.0224}, "46edc488a4": {"quality": 0.39885000000000004, "cost": 1.3199999999999999e-06, "time": 0.0176}, "476a12876c": {"quality": 0.4392333333333333, "cost": 1.92e-06, "time": 0.023}, "48043e2304": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "488645cbd9": {"quality": 0.4425, "cost": 7.2e-07, "time": 0.0118}, "49009a3b57": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.022899999999999997}, "4909061216": {"quality": 0.5304500000000001, "cost": 4.07e-06, "time": 0.037500000000000006}, "49107972df": {"quality": 0.390425, "cost": 1.8e-06, "time": 0.026600000000000002}, "49731b1ccd": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "498f146004": {"quality": 0.5715, "cost": 3.3800000000000002e-06, "time": 0.0399}, "49ad844bd2": {"quality": 0.4847, "cost": 3.51e-06, "time": 0.0381}, "49ca727e49": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "4a4a960a82": {"quality": 0.48889999999999995, "cost": 2.24e-06, "time": 0.026199999999999998}, "4a92372986": {"quality": 0.40216666666666673, "cost": 8.4e-07, "time": 0.0149}, "4aa7e8fde6": {"quality": 0.45363333333333333, "cost": 9.9e-07, "time": 0.022600000000000002}, "4aafd39d76": {"quality": 0.5536, "cost": 1.86e-06, "time": 0.022199999999999998}, "4ace1cfad1": {"quality": 0.463975, "cost": 3.44e-06, "time": 0.0341}, "4ad1952206": {"quality": 0.48889999999999995, "cost": 2.24e-06, "time": 0.0262}, "4b59f40131": {"quality": 0.4762, "cost": 2.07e-06, "time": 0.027399999999999997}, "4b86a1c038": {"quality": 0.46642500000000003, "cost": 2.6e-06, "time": 0.0288}, "4b92a26754": {"quality": 0.365, "cost": 1.2e-07, "time": 0.0064}, "4bc4528402": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "4c158a1a4a": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.0197}, "4c954323e3": {"quality": 0.5800000000000001, "cost": 3.62e-06, "time": 0.03609999999999999}, "4d8bcf8ae2": {"quality": 0.365, "cost": 1.2e-07, "time": 0.0064}, "4d91e8a27b": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "4dd3635bc3": {"quality": 0.6403333333333333, "cost": 3.26e-06, "time": 0.0335}, "4dd96bd18f": {"quality": 0.47240000000000004, "cost": 4.28e-06, "time": 0.0427}, "4e298ee0d4": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.029500000000000002}, "4e3443a0f9": {"quality": 0.538875, "cost": 4.9100000000000004e-06, "time": 0.0461}, "4e4b9db2b8": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "4e6509f614": {"quality": 0.43270000000000003, "cost": 2.4e-06, "time": 0.0224}, "4e6a83e751": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "4e8d8e527a": {"quality": 0.38756666666666667, "cost": 1.4399999999999998e-06, "time": 0.024}, "4ef333ab21": {"quality": 0.4166666666666667, "cost": 8.4e-07, "time": 0.0182}, "4f16545711": {"quality": 0.39890000000000003, "cost": 1.6799999999999998e-06, "time": 0.020200000000000003}, "4f8cca1195": {"quality": 0.59795, "cost": 4.63e-06, "time": 0.0435}, "500860eaa2": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "50701b505e": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.026899999999999997}, "50bc87e9cc": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.0229}, "50c03be77c": {"quality": 0.3908333333333333, "cost": 6e-07, "time": 0.0187}, "510375edad": {"quality": 0.382, "cost": 4.8e-07, "time": 0.009000000000000001}, "512fdb607c": {"quality": 0.52195, "cost": 3.830000000000001e-06, "time": 0.0413}, "51583a901c": {"quality": 0.59795, "cost": 4.63e-06, "time": 0.0435}, "521314dab6": {"quality": 0.54595, "cost": 2.96e-06, "time": 0.025099999999999997}, "5241bf401b": {"quality": 0.41340000000000005, "cost": 1.6799999999999998e-06, "time": 0.0235}, "526878b5eb": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "52c1cba6ce": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.040799999999999996}, "52e5d0f4fb": {"quality": 0.3908333333333333, "cost": 6e-07, "time": 0.0187}, "52f041a70e": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "5307496302": {"quality": 0.4762, "cost": 2.0699999999999997e-06, "time": 0.0274}, "53869388bb": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "53d2932c4f": {"quality": 0.5114666666666666, "cost": 3.32e-06, "time": 0.031}, "5474247f91": {"quality": 0.48573333333333335, "cost": 2.48e-06, "time": 0.0191}, "557d2cf7ba": {"quality": 0.48573333333333335, "cost": 2.48e-06, "time": 0.0191}, "559c7120c5": {"quality": 0.513525, "cost": 2.99e-06, "time": 0.0327}, "55c8aa8935": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.0336}, "56a29a28c5": {"quality": 0.41585, "cost": 1.5599999999999999e-06, "time": 0.0138}, "56b39eb1d6": {"quality": 0.5785750000000001, "cost": 4.39e-06, "time": 0.044}, "5703697dbd": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "5718f2ed80": {"quality": 0.5680999999999999, "cost": 1.86e-06, "time": 0.0255}, "572a02a59a": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "572c2df793": {"quality": 0.439975, "cost": 1.3499999999999998e-06, "time": 0.0252}, "5750713a41": {"quality": 0.4392333333333333, "cost": 1.92e-06, "time": 0.023}, "57757ef15e": {"quality": 0.5002333333333333, "cost": 2.48e-06, "time": 0.0224}, "5793d14bbe": {"quality": 0.5314749999999999, "cost": 4.000000000000001e-06, "time": 0.0401}, "579a915ed2": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "579c81bbe0": {"quality": 0.5291, "cost": 2.12e-06, "time": 0.0165}, "57bed1722f": {"quality": 0.41585, "cost": 1.5599999999999999e-06, "time": 0.0138}, "585ba6d20b": {"quality": 0.5328999999999999, "cost": 3.23e-06, "time": 0.0322}, "589a1cea79": {"quality": 0.47492500000000004, "cost": 2.84e-06, "time": 0.025}, "59006532b4": {"quality": 0.49876666666666675, "cost": 3.15e-06, "time": 0.0322}, "59326c4e00": {"quality": 0.5291, "cost": 2.12e-06, "time": 0.0165}, "596f0ed542": {"quality": 0.47485000000000005, "cost": 3.44e-06, "time": 0.0374}, "5971ba4e0d": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "59e0117b7d": {"quality": 0.46642500000000003, "cost": 2.6e-06, "time": 0.0288}, "59f515d0da": {"quality": 0.48889999999999995, "cost": 2.24e-06, "time": 0.0262}, "59f887b67c": {"quality": 0.47492500000000004, "cost": 2.84e-06, "time": 0.025}, "5a22920db4": {"quality": 0.5367, "cost": 1.11e-06, "time": 0.0157}, "5a35020d45": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "5aa71bb88a": {"quality": 0.40375, "cost": 4.8e-07, "time": 0.0123}, "5b10fbdbe1": {"quality": 0.5399750000000001, "cost": 4.24e-06, "time": 0.0363}, "5b4ad39a9e": {"quality": 0.48889999999999995, "cost": 2.24e-06, "time": 0.0262}, "5bade9eb85": {"quality": 0.523375, "cost": 3.06e-06, "time": 0.0334}, "5be16744bf": {"quality": 0.4969666666666666, "cost": 3.32e-06, "time": 0.0277}, "5c0db11303": {"quality": 0.37633333333333335, "cost": 6e-07, "time": 0.0154}, "5c53feccd9": {"quality": 0.48090000000000005, "cost": 4.52e-06, "time": 0.038900000000000004}, "5c77c7c2b2": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "5d072194b8": {"quality": 0.47729999999999995, "cost": 2.6e-06, "time": 0.0321}, "5d79b50feb": {"quality": 0.5724666666666667, "cost": 3.88e-06, "time": 0.030399999999999996}, "5dc216cd6b": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.030999999999999996}, "5dd68c1b8f": {"quality": 0.5367, "cost": 1.11e-06, "time": 0.0157}, "5de4a882c1": {"quality": 0.6497666666666667, "cost": 4.27e-06, "time": 0.037599999999999995}, "5deeeb223f": {"quality": 0.5244, "cost": 2.99e-06, "time": 0.036}, "5e2f03b962": {"quality": 0.40375, "cost": 4.8e-07, "time": 0.0123}, "5ea2fab380": {"quality": 0.4376, "cost": 1.5599999999999999e-06, "time": 0.0171}, "5eb3bb525b": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.022399999999999996}, "5ec3832817": {"quality": 0.513525, "cost": 2.99e-06, "time": 0.0327}, "5f0199e07b": {"quality": 0.382, "cost": 4.8e-07, "time": 0.009000000000000001}, "5f37b3902b": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027699999999999995}, "60cb623c53": {"quality": 0.42074999999999996, "cost": 7.2e-07, "time": 0.0085}, "612e546d71": {"quality": 0.40980000000000005, "cost": 2.04e-06, "time": 0.0261}, "6160bfb439": {"quality": 0.40735000000000005, "cost": 2.88e-06, "time": 0.031400000000000004}, "6178f33808": {"quality": 0.4484, "cost": 2.19e-06, "time": 0.033800000000000004}, "619b48dde9": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "6268ac658c": {"quality": 0.5206, "cost": 4.000000000000001e-06, "time": 0.0368}, "628f34aace": {"quality": 0.5064500000000001, "cost": 1.9799999999999997e-06, "time": 0.0286}, "630d1ecda0": {"quality": 0.40137500000000004, "cost": 1.2e-06, "time": 0.0175}, "63a0aaebed": {"quality": 0.51495, "cost": 1.11e-06, "time": 0.0124}, "63f392465f": {"quality": 0.45363333333333333, "cost": 9.9e-07, "time": 0.0226}, "6527f214c3": {"quality": 0.4021666666666666, "cost": 8.4e-07, "time": 0.0149}, "652c0f4bdf": {"quality": 0.47247500000000003, "cost": 3.68e-06, "time": 0.0303}, "6533c85913": {"quality": 0.47382500000000005, "cost": 3.51e-06, "time": 0.0348}, "65627426e0": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "65801893b4": {"quality": 0.463975, "cost": 3.44e-06, "time": 0.0341}, "65b76da9c6": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.022399999999999996}, "65be1c1306": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "65e0216208": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "65eee615d7": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "66277da52f": {"quality": 0.470225, "cost": 1.59e-06, "time": 0.027999999999999997}, "66750c0934": {"quality": 0.6403333333333333, "cost": 3.26e-06, "time": 0.0335}, "66776ec181": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "67632141f6": {"quality": 0.41585, "cost": 1.5599999999999999e-06, "time": 0.0138}, "677deb302a": {"quality": 0.467775, "cost": 2.43e-06, "time": 0.0333}, "67868fcff6": {"quality": 0.5837, "cost": 4.7200000000000005e-06, "time": 0.03899999999999999}, "67aad9ea16": {"quality": 0.45690000000000003, "cost": 2.4299999999999996e-06, "time": 0.03}, "67bab6732d": {"quality": 0.5082000000000001, "cost": 4.16e-06, "time": 0.0363}, "67fe399cf1": {"quality": 0.5147333333333334, "cost": 2.48e-06, "time": 0.025699999999999997}, "6846bd8fb3": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "68583552fb": {"quality": 0.39890000000000003, "cost": 1.6799999999999998e-06, "time": 0.020200000000000003}, "689e327daf": {"quality": 0.382, "cost": 4.8e-07, "time": 0.009000000000000001}, "69b3b67de6": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "69bf3f6ba0": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.0224}, "69f90e610f": {"quality": 0.52195, "cost": 3.830000000000001e-06, "time": 0.0413}, "6a022c3f73": {"quality": 0.42799999999999994, "cost": 1.08e-06, "time": 0.0144}, "6a10c53ad8": {"quality": 0.588425, "cost": 4.46e-06, "time": 0.044700000000000004}, "6a6348f69d": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "6a8726145c": {"quality": 0.4762, "cost": 2.0699999999999997e-06, "time": 0.0274}, "6a8a675442": {"quality": 0.4762, "cost": 2.0699999999999997e-06, "time": 0.0274}, "6aac59742a": {"quality": 0.46540000000000004, "cost": 2.67e-06, "time": 0.0262}, "6ac193c88f": {"quality": 0.420675, "cost": 2.04e-06, "time": 0.0294}, "6ae9e9de0b": {"quality": 0.5244, "cost": 2.99e-06, "time": 0.036}, "6b0c585f5c": {"quality": 0.420675, "cost": 2.04e-06, "time": 0.0294}, "6b3c16def2": {"quality": 0.5367, "cost": 1.11e-06, "time": 0.0157}, "6c05c47050": {"quality": 0.4649666666666667, "cost": 1.23e-06, "time": 0.0188}, "6c3667811b": {"quality": 0.41340000000000005, "cost": 1.6799999999999998e-06, "time": 0.0235}, "6cc813aa68": {"quality": 0.5291, "cost": 2.12e-06, "time": 0.0165}, "6cd78cac7e": {"quality": 0.48890000000000006, "cost": 2.24e-06, "time": 0.026199999999999998}, "6d20c6ace0": {"quality": 0.5724666666666667, "cost": 3.88e-06, "time": 0.0304}, "6d67c56ba6": {"quality": 0.466425, "cost": 2.6e-06, "time": 0.0288}, "6db70dc3b6": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "6e0690f576": {"quality": 0.42074999999999996, "cost": 7.2e-07, "time": 0.0085}, "6e3db7ec5e": {"quality": 0.47485, "cost": 3.44e-06, "time": 0.037399999999999996}, "6e62bbb47f": {"quality": 0.45085000000000003, "cost": 1.35e-06, "time": 0.028499999999999998}, "6e859bfae6": {"quality": 0.439975, "cost": 1.3499999999999998e-06, "time": 0.0252}, "6e93514f45": {"quality": 0.4102333333333333, "cost": 1.92e-06, "time": 0.016399999999999998}, "6eae47102b": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.0263}, "6ecf93c479": {"quality": 0.47946666666666665, "cost": 1.2299999999999999e-06, "time": 0.022099999999999998}, "6ef3b7127e": {"quality": 0.6592, "cost": 3.52e-06, "time": 0.0278}, "6f323f80c7": {"quality": 0.365, "cost": 1.2e-07, "time": 0.0064}, "6f60a05c33": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "6fbdd8b57c": {"quality": 0.4021666666666666, "cost": 8.4e-07, "time": 0.0149}, "6fd6046c4b": {"quality": 0.45935, "cost": 1.5899999999999998e-06, "time": 0.0247}, "6fe0b3f929": {"quality": 0.42146666666666666, "cost": 2.76e-06, "time": 0.025}, "6ff4f667f8": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "700474dfbd": {"quality": 0.47946666666666665, "cost": 1.2299999999999999e-06, "time": 0.022099999999999998}, "700ab1d309": {"quality": 0.48573333333333335, "cost": 2.48e-06, "time": 0.0191}, "7040e83d52": {"quality": 0.6497666666666667, "cost": 4.27e-06, "time": 0.037599999999999995}, "704209377f": {"quality": 0.46777500000000005, "cost": 2.43e-06, "time": 0.033299999999999996}, "70b666e371": {"quality": 0.39890000000000003, "cost": 1.68e-06, "time": 0.020200000000000003}, "70c850e039": {"quality": 0.5318, "cost": 1.95e-06, "time": 0.020999999999999998}, "7112a7e64c": {"quality": 0.64505, "cost": 2.51e-06, "time": 0.0237}, "7114013f0c": {"quality": 0.45363333333333333, "cost": 9.9e-07, "time": 0.0226}, "715070d0ca": {"quality": 0.4098, "cost": 2.04e-06, "time": 0.026099999999999998}, "71b615468b": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027700000000000002}, "71ed893462": {"quality": 0.47485000000000005, "cost": 3.44e-06, "time": 0.0374}, "722d41b2f8": {"quality": 0.3989, "cost": 1.6799999999999998e-06, "time": 0.0202}, "723fd5589a": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "7250da0f41": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027700000000000002}, "7260a96349": {"quality": 0.4013, "cost": 1.8e-06, "time": 0.0299}, "7347cf0308": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "736e652158": {"quality": 0.46785, "cost": 1.8299999999999998e-06, "time": 0.020900000000000002}, "738364d6a2": {"quality": 0.45555, "cost": 2.6e-06, "time": 0.025500000000000002}, "739b1f81dc": {"quality": 0.5367, "cost": 1.11e-06, "time": 0.0157}, "73c6240c29": {"quality": 0.448475, "cost": 1.5899999999999998e-06, "time": 0.021400000000000002}, "73fc2767b9": {"quality": 0.4649666666666667, "cost": 1.2299999999999999e-06, "time": 0.0188}, "7445d99939": {"quality": 0.43596666666666667, "cost": 2.76e-06, "time": 0.0283}, "7466a5f424": {"quality": 0.537525, "cost": 5.0800000000000005e-06, "time": 0.0416}, "74cc4b1bc4": {"quality": 0.4762, "cost": 2.07e-06, "time": 0.027399999999999997}, "74d7f64b8c": {"quality": 0.46785, "cost": 1.8299999999999998e-06, "time": 0.020900000000000002}, "751869cbec": {"quality": 0.513525, "cost": 2.99e-06, "time": 0.0327}, "7524905580": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "752d9649f2": {"quality": 0.5484, "cost": 5.0800000000000005e-06, "time": 0.044899999999999995}, "7558c9722d": {"quality": 0.47485, "cost": 3.44e-06, "time": 0.0374}, "75ca9cd4f8": {"quality": 0.365, "cost": 2.4e-07, "time": 0.0128}, "7604c0aa13": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027699999999999995}, "765dbc6ad5": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "7707e6e7e3": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "7765576286": {"quality": 0.4847, "cost": 3.51e-06, "time": 0.0381}, "77983b6105": {"quality": 0.382, "cost": 4.8e-07, "time": 0.009000000000000001}, "77b5740025": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.022899999999999997}, "77c02b00c1": {"quality": 0.48335, "cost": 3.68e-06, "time": 0.033600000000000005}, "77c6a9703a": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "77f293b737": {"quality": 0.5785750000000001, "cost": 4.39e-06, "time": 0.044}, "7801da66b9": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "7862ea67cb": {"quality": 0.39885000000000004, "cost": 1.3199999999999999e-06, "time": 0.0176}, "786e5d0af5": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "795d119bc7": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "7989343d94": {"quality": 0.4166666666666667, "cost": 8.4e-07, "time": 0.0182}, "79fad58f07": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "7a207b42a8": {"quality": 0.4183, "cost": 2.2799999999999998e-06, "time": 0.0223}, "7a58d3472b": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "7a7cc658c8": {"quality": 0.52195, "cost": 3.830000000000001e-06, "time": 0.041299999999999996}, "7b024a2966": {"quality": 0.513525, "cost": 2.99e-06, "time": 0.0327}, "7b6dc3702e": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "7b6f44618e": {"quality": 0.4917750000000001, "cost": 4.52e-06, "time": 0.0422}, "7b74b23910": {"quality": 0.5611333333333334, "cost": 3.64e-06, "time": 0.034199999999999994}, "7b9cc96081": {"quality": 0.365, "cost": 1.2e-07, "time": 0.0064}, "7c45c61d8d": {"quality": 0.467775, "cost": 2.43e-06, "time": 0.033299999999999996}, "7c89a2b69e": {"quality": 0.53045, "cost": 4.07e-06, "time": 0.0375}, "7d44f0959d": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "7d60c38c5c": {"quality": 0.6309, "cost": 1.5e-06, "time": 0.0196}, "7d67e14414": {"quality": 0.517325, "cost": 1.98e-06, "time": 0.0319}, "7daf7ff182": {"quality": 0.5082000000000001, "cost": 4.16e-06, "time": 0.0363}, "7e0ad1c9c1": {"quality": 0.38756666666666667, "cost": 1.4399999999999998e-06, "time": 0.024}, "7ed07ad40a": {"quality": 0.5648333333333334, "cost": 2.7e-06, "time": 0.030799999999999998}, "7fa67a7656": {"quality": 0.5114666666666666, "cost": 3.32e-06, "time": 0.031}, "806881adcb": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "80a1d9c2f3": {"quality": 0.4021666666666666, "cost": 8.4e-07, "time": 0.0149}, "80be7df955": {"quality": 0.42074999999999996, "cost": 7.2e-07, "time": 0.0085}, "80bf60c422": {"quality": 0.51495, "cost": 1.11e-06, "time": 0.0124}, "81333c7a33": {"quality": 0.6592, "cost": 5.28e-06, "time": 0.0417}, "813e75210b": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "815e7116df": {"quality": 0.37633333333333335, "cost": 6e-07, "time": 0.0154}, "816068ff07": {"quality": 0.439975, "cost": 1.3499999999999998e-06, "time": 0.0252}, "81a4f42fd9": {"quality": 0.5121, "cost": 1.88e-06, "time": 0.0203}, "828ccea2d3": {"quality": 0.48889999999999995, "cost": 2.24e-06, "time": 0.026199999999999998}, "829df73946": {"quality": 0.64505, "cost": 2.51e-06, "time": 0.0237}, "831728b179": {"quality": 0.4101333333333334, "cost": 2.5199999999999996e-06, "time": 0.0288}, "831e8b8be5": {"quality": 0.4794666666666667, "cost": 1.23e-06, "time": 0.022099999999999998}, "8357183895": {"quality": 0.46540000000000004, "cost": 2.67e-06, "time": 0.0262}, "8392a6083a": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.033600000000000005}, "83b244c163": {"quality": 0.5121, "cost": 1.88e-06, "time": 0.0203}, "83c9e66ec6": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "842c0d1062": {"quality": 0.45935000000000004, "cost": 1.5899999999999998e-06, "time": 0.0247}, "846bed2aa7": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "847fd49235": {"quality": 0.42799999999999994, "cost": 1.08e-06, "time": 0.0144}, "84dc98be95": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "8519bef585": {"quality": 0.42074999999999996, "cost": 7.2e-07, "time": 0.0085}, "8572c6af3a": {"quality": 0.5244, "cost": 2.99e-06, "time": 0.036}, "85c94a5505": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "85e8eaed6e": {"quality": 0.41225, "cost": 1.2e-06, "time": 0.0208}, "862183bfb9": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "8668f65f05": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "870e2f87b4": {"quality": 0.587075, "cost": 4.63e-06, "time": 0.0402}, "87c1b31c82": {"quality": 0.463975, "cost": 3.44e-06, "time": 0.0341}, "88436e05a9": {"quality": 0.455475, "cost": 3.2000000000000003e-06, "time": 0.0379}, "887ad124e1": {"quality": 0.5053, "cost": 1.47e-06, "time": 0.021599999999999998}, "8886cb3082": {"quality": 0.5328999999999999, "cost": 3.23e-06, "time": 0.0322}, "8940398bf1": {"quality": 0.49795, "cost": 8.7e-07, "time": 0.0162}, "8941621423": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "8961e4d901": {"quality": 0.4102333333333333, "cost": 1.92e-06, "time": 0.016399999999999998}, "8974aa89a0": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "89a289907e": {"quality": 0.590875, "cost": 3.62e-06, "time": 0.0394}, "89a35a09b1": {"quality": 0.48714999999999997, "cost": 2.67e-06, "time": 0.0328}, "89bc21961a": {"quality": 0.365, "cost": 2.4e-07, "time": 0.0128}, "89fbefd150": {"quality": 0.47485, "cost": 3.44e-06, "time": 0.0374}, "8a37c82283": {"quality": 0.5611333333333334, "cost": 3.64e-06, "time": 0.034199999999999994}, "8aaadb8649": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "8acd758b7f": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "8b721bbc6f": {"quality": 0.48714999999999997, "cost": 2.67e-06, "time": 0.0328}, "8b90f4b639": {"quality": 0.4101333333333334, "cost": 2.5199999999999996e-06, "time": 0.0288}, "8bbbe0f52a": {"quality": 0.4392333333333333, "cost": 1.92e-06, "time": 0.023}, "8bc184f385": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.026899999999999997}, "8bf5c3eadc": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "8c195addc7": {"quality": 0.5611333333333334, "cost": 3.64e-06, "time": 0.034199999999999994}, "8c9881972c": {"quality": 0.4134, "cost": 1.68e-06, "time": 0.0235}, "8cf8b81d84": {"quality": 0.38756666666666667, "cost": 1.4399999999999998e-06, "time": 0.024}, "8d79e03266": {"quality": 0.6497666666666667, "cost": 4.27e-06, "time": 0.037599999999999995}, "8d90814b94": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "8e33fac90f": {"quality": 0.47485, "cost": 3.44e-06, "time": 0.0374}, "8e5842ccbd": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.026899999999999997}, "8f4caddfe6": {"quality": 0.5869666666666667, "cost": 3.88e-06, "time": 0.033699999999999994}, "8f4edde3f0": {"quality": 0.4857333333333333, "cost": 2.48e-06, "time": 0.0191}, "900a58f984": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "9025e2480f": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "9028588af4": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "9059fd80ad": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "90d5e40c1b": {"quality": 0.42146666666666666, "cost": 2.76e-06, "time": 0.025}, "90d9a86a2a": {"quality": 0.49876666666666675, "cost": 3.15e-06, "time": 0.0322}, "90ed9312e1": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.0229}, "90ff13783c": {"quality": 0.5304500000000001, "cost": 4.07e-06, "time": 0.0375}, "90ff8eb055": {"quality": 0.426725, "cost": 3.1199999999999998e-06, "time": 0.030900000000000004}, "9104e31369": {"quality": 0.53045, "cost": 4.07e-06, "time": 0.0375}, "918983323f": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "91beb0cac1": {"quality": 0.382, "cost": 4.8e-07, "time": 0.009000000000000001}, "91c800af6b": {"quality": 0.47382500000000005, "cost": 3.51e-06, "time": 0.0348}, "91dd8884db": {"quality": 0.46642500000000003, "cost": 2.6e-06, "time": 0.0288}, "924f128b3c": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.0229}, "9288642e53": {"quality": 0.5291, "cost": 2.12e-06, "time": 0.0165}, "92c4137fb1": {"quality": 0.46777500000000005, "cost": 2.43e-06, "time": 0.0333}, "92c9dcd43b": {"quality": 0.543775, "cost": 3.23e-06, "time": 0.0355}, "92f45e5cc7": {"quality": 0.49795, "cost": 8.7e-07, "time": 0.0162}, "93011c0821": {"quality": 0.53045, "cost": 4.07e-06, "time": 0.0375}, "9303149ba4": {"quality": 0.5050250000000001, "cost": 2.7500000000000004e-06, "time": 0.0365}, "933b4d17dd": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "94010928c6": {"quality": 0.42799999999999994, "cost": 1.08e-06, "time": 0.0144}, "9403809e44": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "943baaea0c": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "94569f177a": {"quality": 0.4649666666666667, "cost": 1.23e-06, "time": 0.018799999999999997}, "9466542023": {"quality": 0.3876666666666666, "cost": 8.4e-07, "time": 0.0116}, "947e28ef2e": {"quality": 0.4166666666666667, "cost": 8.4e-07, "time": 0.0182}, "94ac356663": {"quality": 0.5114666666666666, "cost": 3.32e-06, "time": 0.031}, "94dff9a424": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "9508356a2e": {"quality": 0.4763, "cost": 1.47e-06, "time": 0.015}, "956bdcc254": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.0224}, "9594b0c783": {"quality": 0.5053, "cost": 1.47e-06, "time": 0.021599999999999998}, "964c671f18": {"quality": 0.587075, "cost": 4.63e-06, "time": 0.0402}, "9679fe2b69": {"quality": 0.5053, "cost": 1.47e-06, "time": 0.021599999999999998}, "968fc95038": {"quality": 0.5053, "cost": 1.47e-06, "time": 0.0216}, "96b487c724": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "96e85f9af4": {"quality": 0.38766666666666666, "cost": 8.4e-07, "time": 0.011600000000000001}, "96f87d6483": {"quality": 0.53045, "cost": 4.07e-06, "time": 0.0375}, "972c83b002": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.023599999999999996}, "975bc44958": {"quality": 0.4794666666666667, "cost": 1.23e-06, "time": 0.022099999999999998}, "977a4d6b6b": {"quality": 0.5484, "cost": 5.0800000000000005e-06, "time": 0.044899999999999995}, "97ad4cd41a": {"quality": 0.513525, "cost": 2.99e-06, "time": 0.0327}, "97bc30bd83": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "97e1d0db92": {"quality": 0.39890000000000003, "cost": 1.68e-06, "time": 0.020200000000000003}, "981da9ba40": {"quality": 0.45690000000000003, "cost": 2.43e-06, "time": 0.03}, "98c1ea89f3": {"quality": 0.45935000000000004, "cost": 1.5899999999999998e-06, "time": 0.0247}, "98eca2c65c": {"quality": 0.4630666666666667, "cost": 2e-06, "time": 0.0267}, "98ecf1a157": {"quality": 0.49795, "cost": 8.7e-07, "time": 0.0162}, "9927dc270b": {"quality": 0.5244, "cost": 2.99e-06, "time": 0.036}, "99546d91e4": {"quality": 0.5422666666666667, "cost": 1.6200000000000002e-06, "time": 0.026}, "99cb0ba736": {"quality": 0.4134, "cost": 1.68e-06, "time": 0.0235}, "99e44ab9b2": {"quality": 0.40980000000000005, "cost": 2.04e-06, "time": 0.026099999999999998}, "9a0145c9b5": {"quality": 0.4794666666666667, "cost": 1.23e-06, "time": 0.022099999999999998}, "9a57ea3f89": {"quality": 0.40980000000000005, "cost": 2.04e-06, "time": 0.0261}, "9a8420a0b3": {"quality": 0.4098, "cost": 2.04e-06, "time": 0.026099999999999998}, "9aa32e6c96": {"quality": 0.47485, "cost": 3.44e-06, "time": 0.0374}, "9aa4abfb50": {"quality": 0.39899999999999997, "cost": 7.2e-07, "time": 0.0052}, "9ada932bf5": {"quality": 0.5244, "cost": 2.99e-06, "time": 0.036}, "9b6d4915f3": {"quality": 0.43270000000000003, "cost": 3.6e-06, "time": 0.0336}, "9bae5bafc1": {"quality": 0.5869666666666667, "cost": 3.88e-06, "time": 0.033699999999999994}, "9c549db0a7": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "9c595a2bc9": {"quality": 0.466425, "cost": 2.6e-06, "time": 0.0288}, "9c85f8cfcb": {"quality": 0.51495, "cost": 1.11e-06, "time": 0.0124}, "9c8cc46e6c": {"quality": 0.51495, "cost": 1.11e-06, "time": 0.0124}, "9c97d35a30": {"quality": 0.5680999999999999, "cost": 1.86e-06, "time": 0.0255}, "9ca354a53e": {"quality": 0.39890000000000003, "cost": 1.68e-06, "time": 0.020200000000000003}, "9ce2c3fd98": {"quality": 0.4135, "cost": 1.08e-06, "time": 0.0111}, "9d18cd0737": {"quality": 0.5318, "cost": 1.95e-06, "time": 0.020999999999999998}, "9d7142e7b4": {"quality": 0.5244, "cost": 2.99e-06, "time": 0.036}, "9d778daa24": {"quality": 0.467775, "cost": 2.43e-06, "time": 0.0333}, "9e06360bc9": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "9fb157be35": {"quality": 0.48573333333333335, "cost": 2.48e-06, "time": 0.0191}, "9fc44fdeb1": {"quality": 0.45555, "cost": 2.6e-06, "time": 0.025500000000000002}, "9ffaa26d5a": {"quality": 0.48890000000000006, "cost": 2.24e-06, "time": 0.026199999999999998}, "a041e7777a": {"quality": 0.41340000000000005, "cost": 1.6799999999999998e-06, "time": 0.0235}, "a0b81be5b4": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "a0c85d260e": {"quality": 0.5955, "cost": 5.47e-06, "time": 0.048799999999999996}, "a0dc9f50ac": {"quality": 0.43596666666666667, "cost": 2.76e-06, "time": 0.0283}, "a14c507393": {"quality": 0.4134, "cost": 1.68e-06, "time": 0.0235}, "a1881eb481": {"quality": 0.46777500000000005, "cost": 2.43e-06, "time": 0.0333}, "a1bb32e6a1": {"quality": 0.4021666666666666, "cost": 8.4e-07, "time": 0.0149}, "a2347e8e9e": {"quality": 0.4794666666666667, "cost": 1.23e-06, "time": 0.022099999999999998}, "a2aa082d14": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027700000000000002}, "a2cd339ad9": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.040799999999999996}, "a2fd03e6a5": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "a31e87d7cb": {"quality": 0.4425, "cost": 7.2e-07, "time": 0.0118}, "a344b2d79a": {"quality": 0.52195, "cost": 3.830000000000001e-06, "time": 0.0413}, "a3c0ea3342": {"quality": 0.47729999999999995, "cost": 2.6e-06, "time": 0.0321}, "a456d75fef": {"quality": 0.38766666666666666, "cost": 8.4e-07, "time": 0.011600000000000001}, "a457f6c300": {"quality": 0.4135, "cost": 1.08e-06, "time": 0.0111}, "a4767e7679": {"quality": 0.39890000000000003, "cost": 1.68e-06, "time": 0.020200000000000003}, "a47de025c8": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "a4ad96343d": {"quality": 0.46642500000000003, "cost": 2.6e-06, "time": 0.0288}, "a515a9c8cc": {"quality": 0.4376, "cost": 1.5599999999999999e-06, "time": 0.0171}, "a5949b76ec": {"quality": 0.42146666666666666, "cost": 2.76e-06, "time": 0.025}, "a5ae4dfe66": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.022899999999999997}, "a60dd076b8": {"quality": 0.4392333333333333, "cost": 1.92e-06, "time": 0.023}, "a6297a6c56": {"quality": 0.4376, "cost": 1.5599999999999999e-06, "time": 0.0171}, "a6460dbb7c": {"quality": 0.64505, "cost": 2.51e-06, "time": 0.0237}, "a6796ed686": {"quality": 0.45085000000000003, "cost": 1.35e-06, "time": 0.028499999999999998}, "a6d2b05ec8": {"quality": 0.5715, "cost": 3.3800000000000002e-06, "time": 0.0399}, "a717c4c535": {"quality": 0.4917750000000001, "cost": 4.52e-06, "time": 0.0422}, "a721cd9ebf": {"quality": 0.48889999999999995, "cost": 2.24e-06, "time": 0.026199999999999998}, "a8090787b1": {"quality": 0.45935000000000004, "cost": 1.5899999999999998e-06, "time": 0.0247}, "a854343d46": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.022899999999999997}, "a86b137d7f": {"quality": 0.5508500000000001, "cost": 2.12e-06, "time": 0.019799999999999998}, "a88eb1493c": {"quality": 0.5318, "cost": 1.95e-06, "time": 0.020999999999999998}, "a88fb984e3": {"quality": 0.52195, "cost": 3.830000000000001e-06, "time": 0.0413}, "a94e2e5f57": {"quality": 0.4630666666666667, "cost": 2e-06, "time": 0.026699999999999998}, "a95b4a6dd0": {"quality": 0.42146666666666666, "cost": 2.76e-06, "time": 0.025}, "a9621ea4e6": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.0197}, "a96e22379d": {"quality": 0.46777500000000005, "cost": 2.43e-06, "time": 0.033299999999999996}, "a9721a0a50": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.030999999999999996}, "aa08180e36": {"quality": 0.4858, "cost": 2.84e-06, "time": 0.0283}, "aa38702a02": {"quality": 0.42799999999999994, "cost": 1.08e-06, "time": 0.0144}, "aa8187c023": {"quality": 0.39890000000000003, "cost": 1.6799999999999998e-06, "time": 0.020200000000000003}, "aadbfc418b": {"quality": 0.44250000000000006, "cost": 1.08e-06, "time": 0.0177}, "ab1c706436": {"quality": 0.46532500000000004, "cost": 3.27e-06, "time": 0.0386}, "ab43b02cb0": {"quality": 0.5082000000000001, "cost": 4.16e-06, "time": 0.0363}, "aba21780bc": {"quality": 0.39885000000000004, "cost": 1.3199999999999999e-06, "time": 0.0176}, "abbca95f00": {"quality": 0.5121, "cost": 1.88e-06, "time": 0.0203}, "ac208e7a1d": {"quality": 0.5724666666666667, "cost": 3.88e-06, "time": 0.0304}, "ac7fcf90e2": {"quality": 0.4098, "cost": 2.04e-06, "time": 0.0261}, "ac828ffe70": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "ac9fdc1550": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "ad328d5108": {"quality": 0.365, "cost": 3.5999999999999994e-07, "time": 0.019200000000000002}, "ad3efe44c3": {"quality": 0.39899999999999997, "cost": 7.2e-07, "time": 0.0052}, "ad48432c22": {"quality": 0.46540000000000004, "cost": 2.67e-06, "time": 0.0262}, "ad5187a390": {"quality": 0.463975, "cost": 3.44e-06, "time": 0.034100000000000005}, "ad6ebbba8d": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "ad90055ef6": {"quality": 0.39899999999999997, "cost": 7.2e-07, "time": 0.0052}, "ad97c5cee6": {"quality": 0.3908333333333333, "cost": 6e-07, "time": 0.0187}, "adab1e0fb1": {"quality": 0.5508500000000001, "cost": 2.12e-06, "time": 0.019799999999999998}, "ae655ec593": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "ae94b172be": {"quality": 0.5002333333333333, "cost": 2.48e-06, "time": 0.0224}, "af360c323c": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.018299999999999997}, "af90567194": {"quality": 0.587075, "cost": 4.63e-06, "time": 0.0402}, "b0156bb6d2": {"quality": 0.47485, "cost": 3.44e-06, "time": 0.0374}, "b03c31ca45": {"quality": 0.514875, "cost": 2.82e-06, "time": 0.0372}, "b0530b98c3": {"quality": 0.46532500000000004, "cost": 3.27e-06, "time": 0.0386}, "b0948c05b6": {"quality": 0.5837, "cost": 4.7200000000000005e-06, "time": 0.03899999999999999}, "b18168b9c1": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "b1cf8d33e5": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "b1dcd7aa24": {"quality": 0.513525, "cost": 2.99e-06, "time": 0.0327}, "b214718d07": {"quality": 0.4649666666666667, "cost": 1.23e-06, "time": 0.0188}, "b28925e4b8": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "b2b057ba41": {"quality": 0.5680999999999999, "cost": 1.86e-06, "time": 0.0255}, "b2e063499d": {"quality": 0.6497666666666667, "cost": 4.27e-06, "time": 0.037599999999999995}, "b3369775dc": {"quality": 0.588425, "cost": 4.46e-06, "time": 0.044700000000000004}, "b35bf038c2": {"quality": 0.37633333333333335, "cost": 6e-07, "time": 0.0154}, "b363b25367": {"quality": 0.40980000000000005, "cost": 2.04e-06, "time": 0.0261}, "b3decd5c2f": {"quality": 0.38756666666666667, "cost": 1.4399999999999998e-06, "time": 0.024}, "b3f20b706d": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "b4002173ee": {"quality": 0.43270000000000003, "cost": 2.4e-06, "time": 0.0224}, "b45fc30d81": {"quality": 0.45690000000000003, "cost": 2.43e-06, "time": 0.03}, "b4a259f6dd": {"quality": 0.44705, "cost": 2.3600000000000003e-06, "time": 0.0293}, "b52cdb3c6d": {"quality": 0.4794666666666667, "cost": 1.23e-06, "time": 0.0221}, "b56c312eda": {"quality": 0.4847, "cost": 3.51e-06, "time": 0.0381}, "b5a02bb8ab": {"quality": 0.40980000000000005, "cost": 2.04e-06, "time": 0.026099999999999998}, "b5e2b41c1c": {"quality": 0.5536, "cost": 1.86e-06, "time": 0.022199999999999998}, "b64ddb14f9": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "b66118d5f2": {"quality": 0.45690000000000003, "cost": 2.43e-06, "time": 0.03}, "b67107a43e": {"quality": 0.46785, "cost": 1.8299999999999998e-06, "time": 0.020900000000000002}, "b682a23b89": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "b690a1ddd6": {"quality": 0.40137500000000004, "cost": 1.2e-06, "time": 0.0175}, "b796b7ffd3": {"quality": 0.590875, "cost": 3.62e-06, "time": 0.0394}, "b7d0e8557f": {"quality": 0.48714999999999997, "cost": 2.67e-06, "time": 0.0328}, "b7f203a0bf": {"quality": 0.467775, "cost": 2.43e-06, "time": 0.0333}, "b81d5b2bd9": {"quality": 0.5244, "cost": 2.99e-06, "time": 0.036}, "b8343f05e1": {"quality": 0.3989, "cost": 1.6799999999999998e-06, "time": 0.0202}, "b8ab3d2f25": {"quality": 0.5869666666666667, "cost": 3.88e-06, "time": 0.033699999999999994}, "b8b569172f": {"quality": 0.4969666666666666, "cost": 3.32e-06, "time": 0.0277}, "b8b91e375d": {"quality": 0.5290250000000001, "cost": 4.84e-06, "time": 0.045399999999999996}, "b8c685904d": {"quality": 0.4762, "cost": 2.07e-06, "time": 0.0274}, "b8d1903276": {"quality": 0.4649666666666667, "cost": 1.23e-06, "time": 0.018799999999999997}, "b91e7fdb29": {"quality": 0.4183, "cost": 2.2799999999999998e-06, "time": 0.0223}, "b9770c2261": {"quality": 0.6309, "cost": 1.5e-06, "time": 0.0196}, "b9bb1e6f8d": {"quality": 0.588425, "cost": 4.46e-06, "time": 0.044700000000000004}, "b9da208432": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "ba3223f6ac": {"quality": 0.428, "cost": 1.08e-06, "time": 0.0144}, "bac3d23c31": {"quality": 0.5206, "cost": 4.000000000000001e-06, "time": 0.0368}, "bb3ee18de1": {"quality": 0.48890000000000006, "cost": 2.24e-06, "time": 0.026199999999999998}, "bb6536b0ab": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.030999999999999996}, "bb70f60bf1": {"quality": 0.40216666666666673, "cost": 8.4e-07, "time": 0.0149}, "bbba9dd6ae": {"quality": 0.64505, "cost": 2.51e-06, "time": 0.0237}, "bbfba2f2ee": {"quality": 0.38766666666666666, "cost": 8.4e-07, "time": 0.011600000000000001}, "bc3d02f753": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "bc4c1fcc64": {"quality": 0.64505, "cost": 2.51e-06, "time": 0.0237}, "bc60556255": {"quality": 0.365, "cost": 1.2e-07, "time": 0.0064}, "bcae7c2fc4": {"quality": 0.4762, "cost": 2.0699999999999997e-06, "time": 0.0274}, "bcef42e3b0": {"quality": 0.4166666666666667, "cost": 8.4e-07, "time": 0.0182}, "bcf4bf7c35": {"quality": 0.40735000000000005, "cost": 2.88e-06, "time": 0.031400000000000004}, "bcfb273436": {"quality": 0.5244, "cost": 2.99e-06, "time": 0.036}, "bd99b2fb21": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.031}, "bdf497196b": {"quality": 0.5121, "cost": 1.88e-06, "time": 0.0203}, "be2ae88f70": {"quality": 0.4135, "cost": 1.08e-06, "time": 0.0111}, "be4740f38f": {"quality": 0.5837, "cost": 4.7200000000000005e-06, "time": 0.03899999999999999}, "bed888d4dc": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "bf2a5d2680": {"quality": 0.4134, "cost": 1.6799999999999998e-06, "time": 0.0235}, "bf7b0a8dc1": {"quality": 0.45792499999999997, "cost": 2.3600000000000003e-06, "time": 0.0326}, "bfed7670ed": {"quality": 0.4969666666666666, "cost": 3.32e-06, "time": 0.0277}, "c06b118e65": {"quality": 0.4762, "cost": 2.0699999999999997e-06, "time": 0.0274}, "c08a5ad170": {"quality": 0.466425, "cost": 2.6e-06, "time": 0.0288}, "c0d53a20de": {"quality": 0.514875, "cost": 2.82e-06, "time": 0.0372}, "c10e588987": {"quality": 0.40980000000000005, "cost": 2.04e-06, "time": 0.026099999999999998}, "c127509a7a": {"quality": 0.6403333333333333, "cost": 3.26e-06, "time": 0.0335}, "c13682c7c7": {"quality": 0.494225, "cost": 3.68e-06, "time": 0.0369}, "c13d6e78e9": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "c145482664": {"quality": 0.47485000000000005, "cost": 3.44e-06, "time": 0.0374}, "c14ff3144d": {"quality": 0.4425, "cost": 7.2e-07, "time": 0.0118}, "c186182658": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "c263c65d0a": {"quality": 0.5064500000000001, "cost": 1.9799999999999997e-06, "time": 0.0286}, "c2949aa902": {"quality": 0.5648333333333334, "cost": 2.7e-06, "time": 0.030799999999999998}, "c31c9d4d8c": {"quality": 0.4649666666666667, "cost": 1.2299999999999999e-06, "time": 0.0188}, "c31e956b35": {"quality": 0.523375, "cost": 3.06e-06, "time": 0.0334}, "c339463a25": {"quality": 0.37633333333333335, "cost": 6e-07, "time": 0.0154}, "c36b525dde": {"quality": 0.5648333333333334, "cost": 2.7e-06, "time": 0.030799999999999998}, "c3d20f33bf": {"quality": 0.5314749999999999, "cost": 4.000000000000001e-06, "time": 0.0401}, "c3ec2cec59": {"quality": 0.5724666666666667, "cost": 3.88e-06, "time": 0.0304}, "c443a2c1fb": {"quality": 0.4098, "cost": 2.04e-06, "time": 0.0261}, "c4a80d19b3": {"quality": 0.5800000000000001, "cost": 3.62e-06, "time": 0.03609999999999999}, "c4c2826afd": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "c4c94a5527": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "c4f3e7665d": {"quality": 0.4135, "cost": 1.08e-06, "time": 0.0111}, "c58e9652b7": {"quality": 0.39287500000000003, "cost": 9.6e-07, "time": 0.0213}, "c5a16b834a": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "c5fbe2076f": {"quality": 0.537525, "cost": 5.0800000000000005e-06, "time": 0.0416}, "c6198f364e": {"quality": 0.513525, "cost": 2.99e-06, "time": 0.0327}, "c691570715": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.0229}, "c691a29c42": {"quality": 0.48090000000000005, "cost": 4.52e-06, "time": 0.038900000000000004}, "c6a339987c": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "c6a4d256ce": {"quality": 0.52195, "cost": 3.83e-06, "time": 0.0413}, "c76222087e": {"quality": 0.4630666666666667, "cost": 2e-06, "time": 0.026699999999999998}, "c772ff3704": {"quality": 0.429175, "cost": 2.2799999999999998e-06, "time": 0.0256}, "c7d4ff0c05": {"quality": 0.467775, "cost": 2.43e-06, "time": 0.033299999999999996}, "c823f7ab29": {"quality": 0.4630666666666667, "cost": 2e-06, "time": 0.026699999999999998}, "c82b926689": {"quality": 0.52195, "cost": 3.830000000000001e-06, "time": 0.0413}, "c82f834e85": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "c9320068f9": {"quality": 0.40735000000000005, "cost": 2.88e-06, "time": 0.031400000000000004}, "c935a33384": {"quality": 0.5484, "cost": 5.0800000000000005e-06, "time": 0.044899999999999995}, "c99f3577c7": {"quality": 0.463975, "cost": 3.44e-06, "time": 0.034100000000000005}, "c9d32a0a82": {"quality": 0.466425, "cost": 2.6e-06, "time": 0.0288}, "ca55c36c3f": {"quality": 0.4794666666666667, "cost": 1.23e-06, "time": 0.0221}, "caa7c0bd6b": {"quality": 0.49876666666666675, "cost": 3.15e-06, "time": 0.0322}, "cac6b051e9": {"quality": 0.53425, "cost": 3.06e-06, "time": 0.036699999999999997}, "cb19d631b2": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "cbb25fb322": {"quality": 0.39287500000000003, "cost": 9.6e-07, "time": 0.0213}, "cbb5eb0e74": {"quality": 0.53425, "cost": 3.06e-06, "time": 0.036699999999999997}, "cbc32cbeff": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.029500000000000002}, "cbd4461293": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "cbe2318045": {"quality": 0.5147333333333334, "cost": 2.48e-06, "time": 0.025699999999999997}, "cc20ebc768": {"quality": 0.47485, "cost": 3.44e-06, "time": 0.0374}, "cc886fe337": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "ccf72745c1": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027700000000000002}, "cd1d418732": {"quality": 0.5082000000000001, "cost": 4.16e-06, "time": 0.0363}, "cd23c79db1": {"quality": 0.48573333333333335, "cost": 2.48e-06, "time": 0.0191}, "cd64fbfcd9": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.023599999999999996}, "cda3e2a4e9": {"quality": 0.4134, "cost": 1.68e-06, "time": 0.0235}, "cdc9ce922f": {"quality": 0.4649666666666667, "cost": 1.23e-06, "time": 0.0188}, "cdf0df2f51": {"quality": 0.45690000000000003, "cost": 2.4299999999999996e-06, "time": 0.03}, "ce281875b4": {"quality": 0.4794666666666667, "cost": 1.23e-06, "time": 0.022099999999999998}, "ce4bc5f348": {"quality": 0.42074999999999996, "cost": 7.2e-07, "time": 0.0085}, "ce980cf86f": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "cecca90dd2": {"quality": 0.5536, "cost": 1.86e-06, "time": 0.022199999999999998}, "cece83de2d": {"quality": 0.4101333333333334, "cost": 2.5199999999999996e-06, "time": 0.0288}, "cf51e0a888": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.0229}, "cf9538faf0": {"quality": 0.5147333333333334, "cost": 2.48e-06, "time": 0.0257}, "cf9d2e224c": {"quality": 0.5508500000000001, "cost": 2.12e-06, "time": 0.019799999999999998}, "cfd36f3a8c": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.0263}, "cffa29a6ef": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "d03596c3de": {"quality": 0.5328999999999999, "cost": 3.23e-06, "time": 0.0322}, "d0f9633442": {"quality": 0.4102333333333333, "cost": 1.92e-06, "time": 0.016399999999999998}, "d192872a51": {"quality": 0.47240000000000004, "cost": 4.28e-06, "time": 0.0427}, "d1d953cac7": {"quality": 0.3989, "cost": 1.6799999999999998e-06, "time": 0.0202}, "d2164c8c4c": {"quality": 0.5611333333333334, "cost": 3.64e-06, "time": 0.034199999999999994}, "d216eab7d8": {"quality": 0.5869666666666667, "cost": 3.88e-06, "time": 0.033699999999999994}, "d266c19ac8": {"quality": 0.525825, "cost": 2.22e-06, "time": 0.0281}, "d2af24b59e": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.0224}, "d3a2d50bd7": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "d3db4cf84d": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "d40174fb0b": {"quality": 0.39885000000000004, "cost": 1.3199999999999999e-06, "time": 0.0176}, "d43fafa19e": {"quality": 0.41585, "cost": 1.5599999999999999e-06, "time": 0.0138}, "d48ead13da": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.022399999999999996}, "d5016f4538": {"quality": 0.41585, "cost": 1.5599999999999999e-06, "time": 0.0138}, "d55e983189": {"quality": 0.5422666666666667, "cost": 1.6200000000000002e-06, "time": 0.026}, "d573c2a414": {"quality": 0.45935000000000004, "cost": 1.59e-06, "time": 0.0247}, "d58036ba66": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.0224}, "d59bacbfe0": {"quality": 0.52195, "cost": 3.830000000000001e-06, "time": 0.041299999999999996}, "d6040140b9": {"quality": 0.5955, "cost": 5.47e-06, "time": 0.048799999999999996}, "d640edd7a7": {"quality": 0.4794666666666667, "cost": 1.23e-06, "time": 0.0221}, "d65185c1a4": {"quality": 0.5680999999999999, "cost": 1.86e-06, "time": 0.0255}, "d690b6d739": {"quality": 0.4649666666666667, "cost": 1.23e-06, "time": 0.018799999999999997}, "d6bd3b66ba": {"quality": 0.4135, "cost": 1.08e-06, "time": 0.0111}, "d6c4e48eeb": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.0263}, "d6c60a5214": {"quality": 0.40375, "cost": 4.8e-07, "time": 0.0123}, "d6cbf265ee": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "d73a9aab4e": {"quality": 0.41585, "cost": 1.5599999999999999e-06, "time": 0.0138}, "d752c30d07": {"quality": 0.5002333333333333, "cost": 2.48e-06, "time": 0.0224}, "d7c0972014": {"quality": 0.5082, "cost": 4.16e-06, "time": 0.0363}, "d7f6c0c9d4": {"quality": 0.4762, "cost": 2.07e-06, "time": 0.0274}, "d813410e44": {"quality": 0.47946666666666665, "cost": 1.2299999999999999e-06, "time": 0.022099999999999998}, "d87eb775da": {"quality": 0.6403333333333333, "cost": 3.26e-06, "time": 0.0335}, "d8bab6c09b": {"quality": 0.53045, "cost": 4.07e-06, "time": 0.0375}, "d8bcac36e8": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027699999999999995}, "d96677d8d4": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.026899999999999997}, "d9e2bb21a3": {"quality": 0.5053, "cost": 1.47e-06, "time": 0.021599999999999998}, "daaadadcc9": {"quality": 0.4858, "cost": 2.84e-06, "time": 0.0283}, "daf855e065": {"quality": 0.525825, "cost": 2.22e-06, "time": 0.0281}, "db6f7259cd": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "db9060cd27": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "dbce95a072": {"quality": 0.5114666666666666, "cost": 3.32e-06, "time": 0.031}, "dbe7d818fa": {"quality": 0.3908333333333333, "cost": 6e-07, "time": 0.0187}, "dc66bccb1c": {"quality": 0.5002333333333333, "cost": 2.48e-06, "time": 0.0224}, "dc90065dea": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "dc9a912501": {"quality": 0.4649666666666667, "cost": 1.2299999999999999e-06, "time": 0.0188}, "dd0d70fedd": {"quality": 0.6309, "cost": 1.5e-06, "time": 0.0196}, "dd76899626": {"quality": 0.463975, "cost": 3.44e-06, "time": 0.0341}, "dd9f5d1ba9": {"quality": 0.45085000000000003, "cost": 1.35e-06, "time": 0.028499999999999998}, "de1645053b": {"quality": 0.5314749999999999, "cost": 4.000000000000001e-06, "time": 0.0401}, "de18bf45e1": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "de1e56370f": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "deb84ddd06": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "df2bb408cf": {"quality": 0.40216666666666673, "cost": 8.4e-07, "time": 0.0149}, "df2ebe2c01": {"quality": 0.45935000000000004, "cost": 1.59e-06, "time": 0.0247}, "dfb8aebe38": {"quality": 0.5244, "cost": 2.99e-06, "time": 0.036}, "dfce6153aa": {"quality": 0.418225, "cost": 2.88e-06, "time": 0.0347}, "dfda94bd2a": {"quality": 0.42074999999999996, "cost": 7.2e-07, "time": 0.0085}, "dff452a9ca": {"quality": 0.4102333333333333, "cost": 1.92e-06, "time": 0.016399999999999998}, "e02f982a26": {"quality": 0.48889999999999995, "cost": 2.24e-06, "time": 0.0262}, "e06701b665": {"quality": 0.4649666666666667, "cost": 1.2299999999999999e-06, "time": 0.0188}, "e18abd2ab0": {"quality": 0.4744, "cost": 2.24e-06, "time": 0.022899999999999997}, "e1e596ee1b": {"quality": 0.46642500000000003, "cost": 2.6e-06, "time": 0.0288}, "e20ba014a1": {"quality": 0.59795, "cost": 4.63e-06, "time": 0.0435}, "e223700849": {"quality": 0.4134, "cost": 1.68e-06, "time": 0.0235}, "e26c7bfbdb": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.0197}, "e35e5f81a7": {"quality": 0.4763, "cost": 1.47e-06, "time": 0.015}, "e3cdc0d870": {"quality": 0.4021666666666666, "cost": 8.4e-07, "time": 0.0149}, "e3df4cf041": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "e3eca9854c": {"quality": 0.41225, "cost": 1.2e-06, "time": 0.0208}, "e47dc3abca": {"quality": 0.5082000000000001, "cost": 4.16e-06, "time": 0.0363}, "e495ff601f": {"quality": 0.466425, "cost": 2.6e-06, "time": 0.0288}, "e4b9d4fb41": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "e510bda989": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "e515bc1935": {"quality": 0.48889999999999995, "cost": 2.24e-06, "time": 0.026199999999999998}, "e517cd2222": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "e51b01f418": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "e520dfae5b": {"quality": 0.47872499999999996, "cost": 1.8299999999999998e-06, "time": 0.0242}, "e53906f84b": {"quality": 0.4649666666666667, "cost": 1.23e-06, "time": 0.0188}, "e53b349cce": {"quality": 0.3989, "cost": 1.6799999999999998e-06, "time": 0.0202}, "e5401ed278": {"quality": 0.463975, "cost": 3.44e-06, "time": 0.034100000000000005}, "e56a16ca66": {"quality": 0.5367, "cost": 1.11e-06, "time": 0.0157}, "e5a2f72b30": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "e5a70a13ac": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "e5fdeb4de9": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "e609601eee": {"quality": 0.39890000000000003, "cost": 1.6799999999999998e-06, "time": 0.020200000000000003}, "e6f141cc8f": {"quality": 0.45690000000000003, "cost": 2.43e-06, "time": 0.030000000000000002}, "e7525c117a": {"quality": 0.4021666666666666, "cost": 8.4e-07, "time": 0.0149}, "e7e94ab7a5": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "e86bd256d6": {"quality": 0.4021666666666666, "cost": 8.4e-07, "time": 0.0149}, "e89283a4d9": {"quality": 0.3908333333333333, "cost": 6e-07, "time": 0.0187}, "e9befb80e0": {"quality": 0.4101333333333334, "cost": 2.5199999999999996e-06, "time": 0.0288}, "ea38031fc1": {"quality": 0.418225, "cost": 2.88e-06, "time": 0.0347}, "ea6ecc5653": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "ea8bcb3ae2": {"quality": 0.5837, "cost": 4.7200000000000005e-06, "time": 0.03899999999999999}, "ea91a2e78b": {"quality": 0.39890000000000003, "cost": 1.68e-06, "time": 0.020200000000000003}, "eac300f0d1": {"quality": 0.45690000000000003, "cost": 2.43e-06, "time": 0.03}, "ebaa9b1297": {"quality": 0.49795, "cost": 8.7e-07, "time": 0.0162}, "ebbe8b6c4f": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.031}, "ebdf3abff2": {"quality": 0.6497666666666667, "cost": 4.27e-06, "time": 0.037599999999999995}, "ebee1f2761": {"quality": 0.46777500000000005, "cost": 2.43e-06, "time": 0.033299999999999996}, "ec8844a5ae": {"quality": 0.365, "cost": 1.2e-07, "time": 0.0064}, "ecb5f78f37": {"quality": 0.522025, "cost": 3.23e-06, "time": 0.028899999999999995}, "ece7ff5129": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "ed60b8cac5": {"quality": 0.40137500000000004, "cost": 1.2e-06, "time": 0.0175}, "ed6b5480a5": {"quality": 0.4376, "cost": 1.5599999999999999e-06, "time": 0.0171}, "eda630dc85": {"quality": 0.6403333333333333, "cost": 3.26e-06, "time": 0.0335}, "edaaee5ed4": {"quality": 0.54595, "cost": 2.96e-06, "time": 0.025099999999999997}, "edb2b764aa": {"quality": 0.5869666666666666, "cost": 3.88e-06, "time": 0.0337}, "edc52339db": {"quality": 0.47247500000000003, "cost": 3.68e-06, "time": 0.0303}, "ee46042c5d": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.0336}, "ee855899d8": {"quality": 0.45363333333333333, "cost": 9.9e-07, "time": 0.0226}, "eef12d478b": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "ef37b3e0be": {"quality": 0.4183, "cost": 2.2799999999999998e-06, "time": 0.0223}, "ef43d497f1": {"quality": 0.48335, "cost": 3.68e-06, "time": 0.033600000000000005}, "ef4d4c4a62": {"quality": 0.5869666666666667, "cost": 3.88e-06, "time": 0.033699999999999994}, "f0655621af": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "f076b4c9ae": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "f07881a734": {"quality": 0.38756666666666667, "cost": 1.44e-06, "time": 0.024}, "f0829510fc": {"quality": 0.418225, "cost": 2.88e-06, "time": 0.0347}, "f11eddb4ed": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "f12622d3d7": {"quality": 0.39892500000000003, "cost": 2.04e-06, "time": 0.0228}, "f1408da253": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "f1770e7d28": {"quality": 0.517325, "cost": 1.98e-06, "time": 0.0319}, "f18cf41929": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "f1bda127f6": {"quality": 0.5114666666666666, "cost": 3.32e-06, "time": 0.031}, "f2a2e91541": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.029500000000000002}, "f2c04ed1c8": {"quality": 0.4376, "cost": 1.5599999999999999e-06, "time": 0.0171}, "f2cf5db12d": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.031}, "f3c7a062f7": {"quality": 0.5422666666666667, "cost": 1.6200000000000002e-06, "time": 0.026}, "f42277df7f": {"quality": 0.49795, "cost": 8.7e-07, "time": 0.0162}, "f437481e3b": {"quality": 0.5399750000000001, "cost": 4.24e-06, "time": 0.0363}, "f487340019": {"quality": 0.4762, "cost": 2.07e-06, "time": 0.0274}, "f497b83523": {"quality": 0.5517, "cost": 2.6300000000000002e-06, "time": 0.0301}, "f4aa8ffdf5": {"quality": 0.40216666666666673, "cost": 8.4e-07, "time": 0.0149}, "f4ab4a73b6": {"quality": 0.48563333333333336, "cost": 3.08e-06, "time": 0.0315}, "f4beb148d0": {"quality": 0.5422666666666667, "cost": 1.6200000000000002e-06, "time": 0.026}, "f4deb72db6": {"quality": 0.4102333333333333, "cost": 1.92e-06, "time": 0.016399999999999998}, "f50a47a0aa": {"quality": 0.47946666666666665, "cost": 1.2299999999999999e-06, "time": 0.022099999999999998}, "f5b9a94dcc": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "f5c27e7172": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.018299999999999997}, "f5e53d963b": {"quality": 0.4376, "cost": 1.5599999999999999e-06, "time": 0.0171}, "f5ecac6743": {"quality": 0.466425, "cost": 2.6e-06, "time": 0.0288}, "f614235c15": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "f627bff3a1": {"quality": 0.47946666666666665, "cost": 1.2299999999999999e-06, "time": 0.022099999999999998}, "f70e54a9a0": {"quality": 0.513525, "cost": 2.99e-06, "time": 0.0327}, "f74c5a862e": {"quality": 0.4098, "cost": 2.04e-06, "time": 0.026099999999999998}, "f74ec023e4": {"quality": 0.494225, "cost": 3.68e-06, "time": 0.0369}, "f783b2b34a": {"quality": 0.4649666666666667, "cost": 1.23e-06, "time": 0.0188}, "f7b048bd54": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "f7c4df993e": {"quality": 0.4763, "cost": 1.47e-06, "time": 0.015}, "f848813dca": {"quality": 0.448475, "cost": 1.5899999999999998e-06, "time": 0.021400000000000002}, "f854533145": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "f8f946b5fb": {"quality": 0.390425, "cost": 1.8e-06, "time": 0.026600000000000002}, "f912592c8d": {"quality": 0.5785750000000001, "cost": 4.39e-06, "time": 0.044}, "f93d9a2693": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "f99096d89c": {"quality": 0.543775, "cost": 3.23e-06, "time": 0.0355}, "f9e8e221f3": {"quality": 0.43596666666666667, "cost": 2.76e-06, "time": 0.0283}, "fa38879eab": {"quality": 0.538875, "cost": 4.9100000000000004e-06, "time": 0.0461}, "fa5b473f15": {"quality": 0.39892500000000003, "cost": 2.04e-06, "time": 0.0228}, "fa7882d46b": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "fa906520d1": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "faabebaa30": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "fb0339a7d0": {"quality": 0.5304500000000001, "cost": 4.07e-06, "time": 0.0375}, "fb0e796cd3": {"quality": 0.38756666666666667, "cost": 1.4399999999999998e-06, "time": 0.024}, "fb216ad6b3": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "fb29372712": {"quality": 0.5715, "cost": 3.3800000000000002e-06, "time": 0.0399}, "fb6216880a": {"quality": 0.522025, "cost": 3.23e-06, "time": 0.028899999999999995}, "fbd6c45271": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "fc1fd5bf54": {"quality": 0.47872499999999996, "cost": 1.8299999999999998e-06, "time": 0.0242}, "fc4832696b": {"quality": 0.41340000000000005, "cost": 1.6799999999999998e-06, "time": 0.0235}, "fccaadfcdb": {"quality": 0.513525, "cost": 2.99e-06, "time": 0.0327}, "fcd3d2b250": {"quality": 0.45690000000000003, "cost": 2.43e-06, "time": 0.030000000000000002}, "fce38334b2": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.033600000000000005}, "fd0709359e": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "fddccfbf94": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.030999999999999996}, "fe2b4d4d8b": {"quality": 0.41225, "cost": 1.2e-06, "time": 0.0208}, "fea4734c09": {"quality": 0.54595, "cost": 2.96e-06, "time": 0.025099999999999997}, "fef1ca27fa": {"quality": 0.4917750000000001, "cost": 4.52e-06, "time": 0.0422}, "ff11cb6a7a": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.0336}, "ff171e34e2": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.023599999999999996}, "ff1c958e21": {"quality": 0.51495, "cost": 1.11e-06, "time": 0.0124}, "ff8e68049a": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}} ================================================ FILE: abacus-research/cheap-priors.json ================================================ {"00c93aec22": {"quality": 0.426725, "cost": 3.1199999999999998e-06, "time": 0.030900000000000004}, "00f4acd0d3": {"quality": 0.49876666666666675, "cost": 3.15e-06, "time": 0.0322}, "0121878170": {"quality": 0.630875, "cost": 2.611e-05, "time": 0.0282}, "01c2f973ad": {"quality": 0.5318, "cost": 1.95e-06, "time": 0.020999999999999998}, "01fca3c717": {"quality": 0.5406666666666666, "cost": 1.4060000000000001e-05, "time": 0.024999999999999998}, "02078988c1": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.031}, "021604dec1": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "0262668df7": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "0267c97b70": {"quality": 0.5648333333333334, "cost": 2.7e-06, "time": 0.0308}, "02d6cdecdc": {"quality": 0.5703, "cost": 1.582e-05, "time": 0.038900000000000004}, "033ca325e6": {"quality": 0.5508500000000001, "cost": 2.12e-06, "time": 0.019799999999999998}, "0364b5e990": {"quality": 0.5594250000000001, "cost": 1.582e-05, "time": 0.0356}, "0375ea52c9": {"quality": 0.7468, "cost": 1.25e-05, "time": 0.0079}, "038a5f0a62": {"quality": 0.5648333333333334, "cost": 2.7e-06, "time": 0.030799999999999998}, "039803b3b1": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.0289}, "03b972cb56": {"quality": 0.5594250000000001, "cost": 1.582e-05, "time": 0.03560000000000001}, "042d933706": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "050b21ce37": {"quality": 0.5161250000000001, "cost": 1.4420000000000001e-05, "time": 0.0309}, "0524f42520": {"quality": 0.63795, "cost": 2.712e-05, "time": 0.032299999999999995}, "05420351e5": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "057e332ab1": {"quality": 0.5594250000000001, "cost": 1.582e-05, "time": 0.0356}, "0646f3f0fb": {"quality": 0.608975, "cost": 1.537e-05, "time": 0.034199999999999994}, "06493715cc": {"quality": 0.4763, "cost": 1.47e-06, "time": 0.015}, "0659531b94": {"quality": 0.6695333333333333, "cost": 1.4000000000000001e-05, "time": 0.0275}, "067ee6e91b": {"quality": 0.49876666666666675, "cost": 3.15e-06, "time": 0.0322}, "0695f9b5fc": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "06e94a0f2e": {"quality": 0.7468, "cost": 1.25e-05, "time": 0.0079}, "073ef31d23": {"quality": 0.5548, "cost": 1.397e-05, "time": 0.026199999999999998}, "078a7e545e": {"quality": 0.6019, "cost": 1.436e-05, "time": 0.0301}, "079feb14a8": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.0263}, "07a3a7daf7": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "08127cd6dd": {"quality": 0.6497666666666667, "cost": 4.27e-06, "time": 0.037599999999999995}, "0833133620": {"quality": 0.5536, "cost": 1.86e-06, "time": 0.022199999999999998}, "087a2cabc4": {"quality": 0.6174000000000001, "cost": 1.6210000000000002e-05, "time": 0.042800000000000005}, "08bf8cc191": {"quality": 0.53045, "cost": 4.07e-06, "time": 0.0375}, "08e1802287": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "08f7f63b30": {"quality": 0.563225, "cost": 1.481e-05, "time": 0.0348}, "090cd3ef31": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "0947216ece": {"quality": 0.5724666666666667, "cost": 3.88e-06, "time": 0.0304}, "096d51f670": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "09791c731b": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "0990c0d4f8": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "0a4c1bbb4a": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.018299999999999997}, "0ac969dde3": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.040799999999999996}, "0b4ab72197": {"quality": 0.42146666666666666, "cost": 2.76e-06, "time": 0.025}, "0bf9d31691": {"quality": 0.5149333333333334, "cost": 1.322e-05, "time": 0.0131}, "0c020b86a3": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.0224}, "0c6c7fe96a": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "0c81c8996a": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "0cd25da9fe": {"quality": 0.5594250000000001, "cost": 1.582e-05, "time": 0.0356}, "0cd78f33d8": {"quality": 0.7081666666666666, "cost": 2.5750000000000002e-05, "time": 0.0256}, "0cdc5954dd": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.031}, "0d53dd53c1": {"quality": 0.5922000000000001, "cost": 2.6560000000000003e-05, "time": 0.0329}, "0d8436af32": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.0289}, "0e38896654": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.0289}, "0ec672e7c8": {"quality": 0.5304500000000001, "cost": 4.07e-06, "time": 0.037500000000000006}, "0ed243f788": {"quality": 0.543775, "cost": 3.23e-06, "time": 0.0355}, "0eeb372802": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.033600000000000005}, "0ef0becc1b": {"quality": 0.6067333333333333, "cost": 1.361e-05, "time": 0.0236}, "0fefead197": {"quality": 0.6417499999999999, "cost": 2.611e-05, "time": 0.0315}, "0ff126ebf8": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "10d1d4bdeb": {"quality": 0.561875, "cost": 1.498e-05, "time": 0.0303}, "114a097c53": {"quality": 0.43270000000000003, "cost": 2.4e-06, "time": 0.0224}, "1175ee37e6": {"quality": 0.5367, "cost": 1.11e-06, "time": 0.0157}, "11bc996d48": {"quality": 0.5648333333333334, "cost": 2.7e-06, "time": 0.030799999999999998}, "11ded03305": {"quality": 0.6067333333333332, "cost": 1.361e-05, "time": 0.0236}, "123fb650fb": {"quality": 0.429175, "cost": 2.2799999999999998e-06, "time": 0.0256}, "132f6f3946": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "133ee5023f": {"quality": 0.54595, "cost": 2.96e-06, "time": 0.025099999999999997}, "13a009fe0c": {"quality": 0.5399750000000001, "cost": 4.24e-06, "time": 0.0363}, "13da306f84": {"quality": 0.4969666666666666, "cost": 3.32e-06, "time": 0.0277}, "13e717e41e": {"quality": 0.59465, "cost": 1.286e-05, "time": 0.0138}, "13f75f9bd0": {"quality": 0.4392333333333333, "cost": 1.92e-06, "time": 0.023}, "140ededb41": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "142e59c03f": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.024399999999999998}, "142f3a7c70": {"quality": 0.4763, "cost": 1.47e-06, "time": 0.015}, "1468dddecc": {"quality": 0.5318, "cost": 1.95e-06, "time": 0.020999999999999998}, "15af009a01": {"quality": 0.5703, "cost": 1.582e-05, "time": 0.038900000000000004}, "15b80a55d3": {"quality": 0.561875, "cost": 1.498e-05, "time": 0.030299999999999997}, "1625e624c5": {"quality": 0.58975, "cost": 1.3700000000000001e-05, "time": 0.0191}, "16cff1c1e9": {"quality": 0.49682499999999996, "cost": 1.358e-05, "time": 0.019000000000000003}, "17407df027": {"quality": 0.5028, "cost": 1.526e-05, "time": 0.0329}, "176da24f53": {"quality": 0.5291, "cost": 2.12e-06, "time": 0.0165}, "179379555f": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "181c91d1be": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.030999999999999996}, "183743e76e": {"quality": 0.43596666666666667, "cost": 2.76e-06, "time": 0.0283}, "186b58c209": {"quality": 0.5161250000000001, "cost": 1.4420000000000001e-05, "time": 0.0309}, "187eace9fe": {"quality": 0.5149333333333334, "cost": 1.322e-05, "time": 0.0131}, "190ed2e1b6": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.0289}, "191aafe1a6": {"quality": 0.6067333333333333, "cost": 1.361e-05, "time": 0.0236}, "194919ad28": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.021699999999999997}, "197bb53f10": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "19ba7d0617": {"quality": 0.7468, "cost": 3.7500000000000003e-05, "time": 0.023700000000000002}, "19e3db7fe7": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.0336}, "1a08cb3f50": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.028900000000000002}, "1a169179f6": {"quality": 0.646375, "cost": 2.7960000000000003e-05, "time": 0.040900000000000006}, "1a71d61ac4": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.0217}, "1ad856985f": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "1adec2dca2": {"quality": 0.47247500000000003, "cost": 3.68e-06, "time": 0.0303}, "1b04a2b184": {"quality": 0.5304500000000001, "cost": 4.07e-06, "time": 0.037500000000000006}, "1b28439bd7": {"quality": 0.5724666666666667, "cost": 3.88e-06, "time": 0.0304}, "1b2c667b15": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "1b4511eada": {"quality": 0.612775, "cost": 1.436e-05, "time": 0.0334}, "1b7e6cad66": {"quality": 0.7030000000000001, "cost": 1.426e-05, "time": 0.0218}, "1beb2fac62": {"quality": 0.4969666666666666, "cost": 3.32e-06, "time": 0.0277}, "1c347e4d91": {"quality": 0.5053, "cost": 1.47e-06, "time": 0.021599999999999998}, "1c35bf4be6": {"quality": 0.610325, "cost": 1.52e-05, "time": 0.0387}, "1c4bbf8f7e": {"quality": 0.630875, "cost": 2.611e-05, "time": 0.0282}, "1c5f1341f6": {"quality": 0.6067333333333333, "cost": 1.361e-05, "time": 0.0236}, "1c71804bec": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.0244}, "1cc6d9efb6": {"quality": 0.5837, "cost": 4.7200000000000005e-06, "time": 0.03899999999999999}, "1ce3d77039": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "1d26090364": {"quality": 0.5318, "cost": 1.95e-06, "time": 0.020999999999999998}, "1d87f97e62": {"quality": 0.6403333333333333, "cost": 3.26e-06, "time": 0.0335}, "1d90fb8ca6": {"quality": 0.7081666666666666, "cost": 2.5750000000000002e-05, "time": 0.0256}, "1da2369719": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "1e18e60895": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "1e1bf7e88b": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "1e8b3521f8": {"quality": 0.5680999999999999, "cost": 1.86e-06, "time": 0.0255}, "1f412964ff": {"quality": 0.6067333333333332, "cost": 1.361e-05, "time": 0.0236}, "1f72cfb78a": {"quality": 0.5703, "cost": 1.582e-05, "time": 0.038900000000000004}, "1fb5d170ad": {"quality": 0.5548, "cost": 1.397e-05, "time": 0.0262}, "20180dd292": {"quality": 0.608975, "cost": 1.537e-05, "time": 0.0342}, "2018bef45f": {"quality": 0.5147333333333334, "cost": 2.48e-06, "time": 0.025699999999999997}, "2075ff1d04": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "208a98f514": {"quality": 0.5800000000000001, "cost": 3.62e-06, "time": 0.03609999999999999}, "20904e5c14": {"quality": 0.5439333333333334, "cost": 1.322e-05, "time": 0.0197}, "20afc3d539": {"quality": 0.7176, "cost": 2.676e-05, "time": 0.0297}, "20e10af7d4": {"quality": 0.47872499999999996, "cost": 1.8299999999999998e-06, "time": 0.0242}, "20e2c0b057": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "211b89b4cd": {"quality": 0.49876666666666675, "cost": 3.15e-06, "time": 0.0322}, "21386082aa": {"quality": 0.561875, "cost": 1.498e-05, "time": 0.0303}, "21b2b8ebd1": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "21b78249a7": {"quality": 0.6308666666666666, "cost": 2.536e-05, "time": 0.0184}, "21bed16a7d": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "2200d969d0": {"quality": 0.695925, "cost": 2.751e-05, "time": 0.03950000000000001}, "220d008704": {"quality": 0.5548, "cost": 1.397e-05, "time": 0.0262}, "2251d21392": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "23566f15ab": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "23a9506d36": {"quality": 0.7468, "cost": 1.25e-05, "time": 0.0079}, "24957f3a43": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.0197}, "2529e2f8b0": {"quality": 0.7468, "cost": 1.25e-05, "time": 0.0079}, "252f01ac5b": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.024399999999999998}, "25fadf0883": {"quality": 0.5703, "cost": 1.582e-05, "time": 0.038900000000000004}, "2609bfd616": {"quality": 0.4858, "cost": 2.84e-06, "time": 0.0283}, "2629f3e324": {"quality": 0.5632250000000001, "cost": 1.481e-05, "time": 0.0348}, "262e4298f9": {"quality": 0.50525, "cost": 1.4420000000000001e-05, "time": 0.0276}, "26cc40d3bb": {"quality": 0.5509999999999999, "cost": 1.498e-05, "time": 0.026999999999999996}, "2728c8eb6a": {"quality": 0.42146666666666666, "cost": 2.76e-06, "time": 0.025}, "27bc52befa": {"quality": 0.5374, "cost": 1.49e-05, "time": 0.0303}, "27daa50458": {"quality": 0.5508500000000001, "cost": 2.12e-06, "time": 0.019799999999999998}, "2821795e69": {"quality": 0.608975, "cost": 1.537e-05, "time": 0.034199999999999994}, "28369b2421": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.0263}, "28421e6d62": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.022399999999999996}, "2936c3e43e": {"quality": 0.5304500000000001, "cost": 4.07e-06, "time": 0.0375}, "293ec5edca": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "29409d0894": {"quality": 0.59465, "cost": 1.286e-05, "time": 0.0138}, "294258298a": {"quality": 0.6067333333333332, "cost": 1.361e-05, "time": 0.0236}, "294e541235": {"quality": 0.51495, "cost": 1.11e-06, "time": 0.0124}, "295ed5e759": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.0197}, "2960431101": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "29892d8468": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.0336}, "299a0aeb65": {"quality": 0.590875, "cost": 3.62e-06, "time": 0.0394}, "29ad99e3ed": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.016399999999999998}, "2a5edac2de": {"quality": 0.5922000000000001, "cost": 2.6560000000000003e-05, "time": 0.0329}, "2a7d15f4a7": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "2aa996de6a": {"quality": 0.5837749999999999, "cost": 2.572e-05, "time": 0.024300000000000002}, "2ac4fb293f": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "2afeff0083": {"quality": 0.563225, "cost": 1.481e-05, "time": 0.034800000000000005}, "2b2bc9568b": {"quality": 0.55235, "cost": 1.481e-05, "time": 0.0315}, "2b5679d248": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "2bcf54cda1": {"quality": 0.50525, "cost": 1.4420000000000001e-05, "time": 0.0276}, "2bd39ee744": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "2bf38d797f": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "2c4f4f304e": {"quality": 0.68885, "cost": 1.325e-05, "time": 0.0177}, "2c5cf9eb26": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.023599999999999996}, "2c9a9f94c4": {"quality": 0.426725, "cost": 3.1199999999999998e-06, "time": 0.030900000000000004}, "2d3bbc2d23": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "2de113167b": {"quality": 0.494225, "cost": 3.68e-06, "time": 0.0369}, "2de3eb2c19": {"quality": 0.6592, "cost": 3.52e-06, "time": 0.0278}, "2e30394ac6": {"quality": 0.4135, "cost": 1.08e-06, "time": 0.011099999999999999}, "2e9c5cc9bf": {"quality": 0.5291, "cost": 2.12e-06, "time": 0.0165}, "2f1573da80": {"quality": 0.429175, "cost": 2.2799999999999998e-06, "time": 0.0256}, "2f39d78f34": {"quality": 0.6453666666666668, "cost": 2.536e-05, "time": 0.0217}, "2fc0cb3592": {"quality": 0.43596666666666667, "cost": 2.76e-06, "time": 0.0283}, "2fd9cd426a": {"quality": 0.48335, "cost": 3.68e-06, "time": 0.033600000000000005}, "300924ebae": {"quality": 0.6308666666666666, "cost": 2.536e-05, "time": 0.0184}, "3019af79b3": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "302c1d97fc": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027699999999999995}, "30ae4cbe91": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "30c1f9ddf1": {"quality": 0.6067333333333333, "cost": 1.361e-05, "time": 0.0236}, "30cd375570": {"quality": 0.6129000000000001, "cost": 1.546e-05, "time": 0.033}, "3169782cbb": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.0164}, "3172fc459a": {"quality": 0.6244750000000001, "cost": 1.722e-05, "time": 0.0469}, "318499c14b": {"quality": 0.610325, "cost": 1.52e-05, "time": 0.0387}, "31a32be94d": {"quality": 0.6393, "cost": 2.695e-05, "time": 0.0368}, "32b101d807": {"quality": 0.538875, "cost": 4.9100000000000004e-06, "time": 0.0461}, "32e2c7ad7f": {"quality": 0.61985, "cost": 1.537e-05, "time": 0.0375}, "33459cd29c": {"quality": 0.47492500000000004, "cost": 2.84e-06, "time": 0.025}, "33a187e74f": {"quality": 0.525825, "cost": 2.22e-06, "time": 0.0281}, "33bab4f766": {"quality": 0.612775, "cost": 1.436e-05, "time": 0.0334}, "34922140da": {"quality": 0.5703, "cost": 1.582e-05, "time": 0.038900000000000004}, "3511b5e1d0": {"quality": 0.39899999999999997, "cost": 1.08e-06, "time": 0.0078}, "3513311c2d": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "3513e54767": {"quality": 0.54595, "cost": 2.96e-06, "time": 0.025099999999999997}, "353f0cb1ac": {"quality": 0.5508500000000001, "cost": 2.12e-06, "time": 0.019799999999999998}, "3550bf88cb": {"quality": 0.53425, "cost": 3.06e-06, "time": 0.036699999999999997}, "35610fb420": {"quality": 0.5536, "cost": 1.86e-06, "time": 0.022199999999999998}, "357267e14b": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "35baa5c3cc": {"quality": 0.61985, "cost": 1.537e-05, "time": 0.0375}, "3637084f91": {"quality": 0.5147333333333334, "cost": 2.48e-06, "time": 0.025699999999999997}, "368a497102": {"quality": 0.5406666666666666, "cost": 1.4060000000000001e-05, "time": 0.025}, "36c66671ee": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.023599999999999996}, "37456cb002": {"quality": 0.55235, "cost": 1.481e-05, "time": 0.0315}, "3746ea5c03": {"quality": 0.5406666666666666, "cost": 1.4060000000000001e-05, "time": 0.024999999999999998}, "375ed248fe": {"quality": 0.5147333333333334, "cost": 2.48e-06, "time": 0.025699999999999997}, "377cdf9209": {"quality": 0.563225, "cost": 1.481e-05, "time": 0.0348}, "37d4d0f214": {"quality": 0.5703, "cost": 1.582e-05, "time": 0.0389}, "37ece7217f": {"quality": 0.59465, "cost": 1.286e-05, "time": 0.0138}, "38075bb01f": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "3831d758b1": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.021699999999999997}, "38567d6a43": {"quality": 0.4763, "cost": 1.47e-06, "time": 0.015}, "3875787727": {"quality": 0.55235, "cost": 1.481e-05, "time": 0.0315}, "389c54cbca": {"quality": 0.54595, "cost": 2.96e-06, "time": 0.025099999999999997}, "3980f20caa": {"quality": 0.48090000000000005, "cost": 4.52e-06, "time": 0.038900000000000004}, "3997a836bd": {"quality": 0.608975, "cost": 1.537e-05, "time": 0.034199999999999994}, "39ad76f8ce": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.024399999999999998}, "39c0b7c171": {"quality": 0.6174000000000001, "cost": 1.621e-05, "time": 0.042800000000000005}, "39cd4ca402": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.018299999999999997}, "3a32c98a53": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "3ac7fa4e46": {"quality": 0.646375, "cost": 2.7960000000000003e-05, "time": 0.040900000000000006}, "3ad6dcf559": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "3ae0de8663": {"quality": 0.5594250000000001, "cost": 1.582e-05, "time": 0.03560000000000001}, "3b2e8075ea": {"quality": 0.5680999999999999, "cost": 1.86e-06, "time": 0.0255}, "3b3676521a": {"quality": 0.5955, "cost": 5.47e-06, "time": 0.048799999999999996}, "3b57530a56": {"quality": 0.64505, "cost": 2.51e-06, "time": 0.0237}, "3b6fbfa11d": {"quality": 0.6129000000000001, "cost": 1.546e-05, "time": 0.033}, "3b81215e7a": {"quality": 0.61985, "cost": 1.537e-05, "time": 0.037500000000000006}, "3b9f8045d7": {"quality": 0.646375, "cost": 2.7960000000000003e-05, "time": 0.040900000000000006}, "3c5857683c": {"quality": 0.5678500000000001, "cost": 1.666e-05, "time": 0.0442}, "3cbab8082e": {"quality": 0.523375, "cost": 3.06e-06, "time": 0.0334}, "3d21104666": {"quality": 0.6421, "cost": 2.62e-05, "time": 0.027}, "3d71c4dd2c": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.026899999999999997}, "3d9e24215e": {"quality": 0.7030000000000001, "cost": 1.426e-05, "time": 0.0218}, "3e7efee65a": {"quality": 0.61985, "cost": 1.537e-05, "time": 0.0375}, "3ed0ad20ed": {"quality": 0.5594250000000001, "cost": 1.582e-05, "time": 0.0356}, "3f1a58aec9": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "3f2b07cb78": {"quality": 0.6129000000000001, "cost": 1.546e-05, "time": 0.033}, "3f3ef494b0": {"quality": 0.43596666666666667, "cost": 2.76e-06, "time": 0.0283}, "3f62c3fbfc": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "3f730d8bfe": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.0244}, "3f8d2ee81f": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "40104c813f": {"quality": 0.522025, "cost": 3.23e-06, "time": 0.028899999999999995}, "403b05da2d": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027699999999999995}, "403f0726fa": {"quality": 0.7468, "cost": 2.5e-05, "time": 0.0158}, "4098178354": {"quality": 0.563225, "cost": 1.481e-05, "time": 0.034800000000000005}, "409ff67607": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.018299999999999997}, "40b3b6642c": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "412c065b83": {"quality": 0.4102333333333333, "cost": 1.92e-06, "time": 0.016399999999999998}, "4191118787": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "41d5b97871": {"quality": 0.43923333333333336, "cost": 1.92e-06, "time": 0.023}, "41d8845655": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "41ee202cac": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "41fe4aee55": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.024399999999999998}, "42430ea391": {"quality": 0.559425, "cost": 1.582e-05, "time": 0.0356}, "42ddd48341": {"quality": 0.537525, "cost": 5.0800000000000005e-06, "time": 0.0416}, "42f1e19aa7": {"quality": 0.63795, "cost": 2.712e-05, "time": 0.032299999999999995}, "430a2ab32f": {"quality": 0.7176, "cost": 2.676e-05, "time": 0.0297}, "4339427ad8": {"quality": 0.513675, "cost": 1.526e-05, "time": 0.0362}, "4361bc7ea7": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "43c3cf9cb8": {"quality": 0.6592, "cost": 3.52e-06, "time": 0.0278}, "43d24fb32a": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027700000000000002}, "43e9b39e5c": {"quality": 0.6740250000000001, "cost": 1.677e-05, "time": 0.0455}, "44d6af5523": {"quality": 0.6309, "cost": 2.25e-06, "time": 0.0294}, "44f189d813": {"quality": 0.55235, "cost": 1.481e-05, "time": 0.0315}, "450f45a187": {"quality": 0.5729, "cost": 1.286e-05, "time": 0.0105}, "453d0a5097": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.0217}, "4547ef4c8e": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "461846a52d": {"quality": 0.42799999999999994, "cost": 1.08e-06, "time": 0.0144}, "462e6ff849": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.0336}, "4630853d32": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "46475b9e75": {"quality": 0.5536, "cost": 1.86e-06, "time": 0.022199999999999998}, "46654a1f32": {"quality": 0.5837, "cost": 4.7200000000000005e-06, "time": 0.039}, "466a3036b2": {"quality": 0.47382500000000005, "cost": 3.51e-06, "time": 0.0348}, "466d4d16dd": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.022399999999999996}, "46ed68152d": {"quality": 0.5002333333333333, "cost": 2.48e-06, "time": 0.0224}, "476a12876c": {"quality": 0.4392333333333333, "cost": 1.92e-06, "time": 0.023}, "4778401a7a": {"quality": 0.50525, "cost": 1.4420000000000001e-05, "time": 0.027600000000000003}, "47f9115b26": {"quality": 0.561875, "cost": 1.498e-05, "time": 0.0303}, "48043e2304": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "487f30e740": {"quality": 0.6129, "cost": 1.546e-05, "time": 0.033}, "488645cbd9": {"quality": 0.4425, "cost": 7.2e-07, "time": 0.0118}, "48bf87f7fe": {"quality": 0.648825, "cost": 2.712e-05, "time": 0.0356}, "4909061216": {"quality": 0.5304500000000001, "cost": 4.07e-06, "time": 0.037500000000000006}, "49731b1ccd": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "49ad844bd2": {"quality": 0.4847, "cost": 3.51e-06, "time": 0.0381}, "49ca727e49": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "4a23d8eff7": {"quality": 0.50525, "cost": 1.4420000000000001e-05, "time": 0.0276}, "4a555da784": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "4a5cea8b85": {"quality": 0.6421, "cost": 2.62e-05, "time": 0.027}, "4a767339bd": {"quality": 0.57275, "cost": 1.498e-05, "time": 0.0336}, "4aafd39d76": {"quality": 0.5536, "cost": 1.86e-06, "time": 0.022199999999999998}, "4aca6e5216": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.0164}, "4b18a647d6": {"quality": 0.630875, "cost": 2.611e-05, "time": 0.0282}, "4bc4528402": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "4c158a1a4a": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.0197}, "4c954323e3": {"quality": 0.5800000000000001, "cost": 3.62e-06, "time": 0.03609999999999999}, "4d91e8a27b": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "4dc185389a": {"quality": 0.5656749999999999, "cost": 1.397e-05, "time": 0.0295}, "4dd3635bc3": {"quality": 0.6403333333333333, "cost": 3.26e-06, "time": 0.0335}, "4dd98ef398": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.021699999999999997}, "4dfacd0007": {"quality": 0.563225, "cost": 1.481e-05, "time": 0.0348}, "4e298ee0d4": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.029500000000000002}, "4e3443a0f9": {"quality": 0.538875, "cost": 4.9100000000000004e-06, "time": 0.0461}, "4e4b9db2b8": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "4e6509f614": {"quality": 0.43270000000000003, "cost": 2.4e-06, "time": 0.0224}, "4e6a83e751": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "4e79c8947f": {"quality": 0.58975, "cost": 1.3700000000000001e-05, "time": 0.0191}, "4e9504432b": {"quality": 0.560775, "cost": 1.565e-05, "time": 0.040100000000000004}, "4e962170dc": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.024399999999999998}, "4eb0826f21": {"quality": 0.5729, "cost": 1.286e-05, "time": 0.0105}, "4ed41bf2e4": {"quality": 0.5837749999999999, "cost": 2.572e-05, "time": 0.024300000000000002}, "4f78672528": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.0164}, "4f8cca1195": {"quality": 0.59795, "cost": 4.63e-06, "time": 0.0435}, "500860eaa2": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "50701b505e": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.026899999999999997}, "51583a901c": {"quality": 0.59795, "cost": 4.63e-06, "time": 0.0435}, "51aeaf9f3e": {"quality": 0.7468, "cost": 1.25e-05, "time": 0.0079}, "520b52b64c": {"quality": 0.7468, "cost": 2.5e-05, "time": 0.0158}, "521314dab6": {"quality": 0.54595, "cost": 2.96e-06, "time": 0.025099999999999997}, "5226eb7ff6": {"quality": 0.6067333333333332, "cost": 1.361e-05, "time": 0.0236}, "526878b5eb": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "52c1cba6ce": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.040799999999999996}, "52f041a70e": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "533867574b": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.0217}, "53869388bb": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "53aefd41e4": {"quality": 0.5439333333333334, "cost": 1.322e-05, "time": 0.019700000000000002}, "53d2932c4f": {"quality": 0.5114666666666666, "cost": 3.32e-06, "time": 0.031}, "54375d3eba": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.0217}, "5474247f91": {"quality": 0.48573333333333335, "cost": 2.48e-06, "time": 0.0191}, "54993bc472": {"quality": 0.59465, "cost": 1.286e-05, "time": 0.0138}, "55358f2285": {"quality": 0.6884, "cost": 1.602e-05, "time": 0.035699999999999996}, "5569b4f878": {"quality": 0.68885, "cost": 1.325e-05, "time": 0.0177}, "557d2cf7ba": {"quality": 0.48573333333333335, "cost": 2.48e-06, "time": 0.0191}, "55c8aa8935": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.0336}, "55e6bf8f14": {"quality": 0.5406666666666666, "cost": 1.4060000000000001e-05, "time": 0.024999999999999998}, "56a0660622": {"quality": 0.58975, "cost": 1.3700000000000001e-05, "time": 0.0191}, "56a29a28c5": {"quality": 0.41585, "cost": 1.5599999999999999e-06, "time": 0.0138}, "56c4fd5056": {"quality": 0.6308666666666667, "cost": 2.536e-05, "time": 0.0184}, "5703697dbd": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "5718f2ed80": {"quality": 0.5680999999999999, "cost": 1.86e-06, "time": 0.0255}, "572a02a59a": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "5750713a41": {"quality": 0.4392333333333333, "cost": 1.92e-06, "time": 0.023}, "57757ef15e": {"quality": 0.5002333333333333, "cost": 2.48e-06, "time": 0.0224}, "579c81bbe0": {"quality": 0.5291, "cost": 2.12e-06, "time": 0.0165}, "57bed1722f": {"quality": 0.41585, "cost": 1.5599999999999999e-06, "time": 0.0138}, "585ba6d20b": {"quality": 0.5328999999999999, "cost": 3.23e-06, "time": 0.0322}, "589267ac64": {"quality": 0.58975, "cost": 1.3700000000000001e-05, "time": 0.0191}, "589a1cea79": {"quality": 0.47492500000000004, "cost": 2.84e-06, "time": 0.025}, "58ca42839b": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.0164}, "58dc373441": {"quality": 0.5076999999999999, "cost": 1.358e-05, "time": 0.0223}, "59006532b4": {"quality": 0.49876666666666675, "cost": 3.15e-06, "time": 0.0322}, "59326c4e00": {"quality": 0.5291, "cost": 2.12e-06, "time": 0.0165}, "593975c75b": {"quality": 0.695925, "cost": 2.751e-05, "time": 0.03950000000000001}, "596b4f8694": {"quality": 0.50525, "cost": 1.4420000000000001e-05, "time": 0.0276}, "5971ba4e0d": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "5996465c0a": {"quality": 0.563225, "cost": 1.481e-05, "time": 0.0348}, "59d70b9f65": {"quality": 0.6695333333333333, "cost": 1.4e-05, "time": 0.0275}, "59f887b67c": {"quality": 0.47492500000000004, "cost": 2.84e-06, "time": 0.025}, "5a22920db4": {"quality": 0.5367, "cost": 1.11e-06, "time": 0.0157}, "5a35020d45": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "5aa43da1fc": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.0244}, "5ae0d88127": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.016399999999999998}, "5b10fbdbe1": {"quality": 0.5399750000000001, "cost": 4.24e-06, "time": 0.0363}, "5bade9eb85": {"quality": 0.523375, "cost": 3.06e-06, "time": 0.0334}, "5be16744bf": {"quality": 0.4969666666666666, "cost": 3.32e-06, "time": 0.0277}, "5c5055e252": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.0164}, "5c53feccd9": {"quality": 0.48090000000000005, "cost": 4.52e-06, "time": 0.038900000000000004}, "5c77c7c2b2": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "5d298b5b48": {"quality": 0.559425, "cost": 1.582e-05, "time": 0.0356}, "5d41515d2e": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "5d4babc723": {"quality": 0.5618749999999999, "cost": 1.498e-05, "time": 0.0303}, "5d79b50feb": {"quality": 0.5724666666666667, "cost": 3.88e-06, "time": 0.030399999999999996}, "5dc216cd6b": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.030999999999999996}, "5dd68c1b8f": {"quality": 0.5367, "cost": 1.11e-06, "time": 0.0157}, "5de4a882c1": {"quality": 0.6497666666666667, "cost": 4.27e-06, "time": 0.037599999999999995}, "5e04e1c72d": {"quality": 0.6453666666666668, "cost": 2.536e-05, "time": 0.0217}, "5e923cee9e": {"quality": 0.55235, "cost": 1.481e-05, "time": 0.0315}, "5ea2fab380": {"quality": 0.4376, "cost": 1.5599999999999999e-06, "time": 0.0171}, "5eb3bb525b": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.022399999999999996}, "5eea899380": {"quality": 0.6174000000000001, "cost": 1.6210000000000002e-05, "time": 0.042800000000000005}, "5f37b3902b": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027699999999999995}, "5f9282df3c": {"quality": 0.50525, "cost": 1.4420000000000001e-05, "time": 0.027600000000000003}, "6019884cf3": {"quality": 0.560775, "cost": 1.565e-05, "time": 0.040100000000000004}, "606352363e": {"quality": 0.5028, "cost": 1.526e-05, "time": 0.0329}, "608728f868": {"quality": 0.55235, "cost": 1.481e-05, "time": 0.0315}, "60b9e936f1": {"quality": 0.5729, "cost": 1.286e-05, "time": 0.0105}, "60cb623c53": {"quality": 0.42074999999999996, "cost": 7.2e-07, "time": 0.0085}, "6234de86b4": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "62352c6854": {"quality": 0.5678500000000001, "cost": 1.666e-05, "time": 0.0442}, "63a0aaebed": {"quality": 0.51495, "cost": 1.11e-06, "time": 0.0124}, "647dda686f": {"quality": 0.61985, "cost": 1.537e-05, "time": 0.0375}, "6511b21ded": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "652c0f4bdf": {"quality": 0.47247500000000003, "cost": 3.68e-06, "time": 0.0303}, "6533c85913": {"quality": 0.47382500000000005, "cost": 3.51e-06, "time": 0.0348}, "65627426e0": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "65b76da9c6": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.022399999999999996}, "65be1c1306": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "65e0216208": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "65eee615d7": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "6623d7a5ac": {"quality": 0.66695, "cost": 1.576e-05, "time": 0.041400000000000006}, "66750c0934": {"quality": 0.6403333333333333, "cost": 3.26e-06, "time": 0.0335}, "66776ec181": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "66e5ae0a21": {"quality": 0.5406666666666667, "cost": 1.4060000000000001e-05, "time": 0.025}, "6750a8d7a7": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "67632141f6": {"quality": 0.41585, "cost": 1.5599999999999999e-06, "time": 0.0138}, "67868fcff6": {"quality": 0.5837, "cost": 4.7200000000000005e-06, "time": 0.03899999999999999}, "67bab6732d": {"quality": 0.5082000000000001, "cost": 4.16e-06, "time": 0.0363}, "67fe399cf1": {"quality": 0.5147333333333334, "cost": 2.48e-06, "time": 0.025699999999999997}, "6846bd8fb3": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "68b4cc3e39": {"quality": 0.581325, "cost": 2.6560000000000003e-05, "time": 0.0296}, "69a029ae36": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.024399999999999998}, "69b3b67de6": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "69bf3f6ba0": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.0224}, "6a022c3f73": {"quality": 0.42799999999999994, "cost": 1.08e-06, "time": 0.0144}, "6a10c53ad8": {"quality": 0.588425, "cost": 4.46e-06, "time": 0.044700000000000004}, "6a6348f69d": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "6a74a11bee": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "6a90ec29dd": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.0244}, "6aac59742a": {"quality": 0.46540000000000004, "cost": 2.67e-06, "time": 0.0262}, "6b0862c597": {"quality": 0.6067333333333332, "cost": 1.361e-05, "time": 0.0236}, "6b3c16def2": {"quality": 0.5367, "cost": 1.11e-06, "time": 0.0157}, "6b99b0e901": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.0244}, "6b9b2b3515": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "6bcc02962b": {"quality": 0.6884, "cost": 1.602e-05, "time": 0.035699999999999996}, "6c1987a9e3": {"quality": 0.57275, "cost": 1.498e-05, "time": 0.0336}, "6c50123ee1": {"quality": 0.68885, "cost": 1.325e-05, "time": 0.0177}, "6c67c36480": {"quality": 0.55235, "cost": 1.481e-05, "time": 0.0315}, "6c9b9f1363": {"quality": 0.610325, "cost": 1.52e-05, "time": 0.0387}, "6cc813aa68": {"quality": 0.5291, "cost": 2.12e-06, "time": 0.0165}, "6d20c6ace0": {"quality": 0.5724666666666667, "cost": 3.88e-06, "time": 0.0304}, "6d444fe21a": {"quality": 0.5406666666666666, "cost": 1.4060000000000001e-05, "time": 0.025}, "6db70dc3b6": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "6e0690f576": {"quality": 0.42074999999999996, "cost": 7.2e-07, "time": 0.0085}, "6e06cc804f": {"quality": 0.6308666666666666, "cost": 2.536e-05, "time": 0.0184}, "6e24048a2e": {"quality": 0.5406666666666666, "cost": 1.4060000000000001e-05, "time": 0.025}, "6e93514f45": {"quality": 0.4102333333333333, "cost": 1.92e-06, "time": 0.016399999999999998}, "6eae47102b": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.0263}, "6ed4cae469": {"quality": 0.7468, "cost": 1.25e-05, "time": 0.0079}, "6ef3b7127e": {"quality": 0.6592, "cost": 3.52e-06, "time": 0.0278}, "6f60a05c33": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "6fe0b3f929": {"quality": 0.42146666666666666, "cost": 2.76e-06, "time": 0.025}, "700ab1d309": {"quality": 0.48573333333333335, "cost": 2.48e-06, "time": 0.0191}, "7040e83d52": {"quality": 0.6497666666666667, "cost": 4.27e-06, "time": 0.037599999999999995}, "7046765af8": {"quality": 0.55235, "cost": 1.481e-05, "time": 0.0315}, "70b7c92ce8": {"quality": 0.5656749999999999, "cost": 1.397e-05, "time": 0.0295}, "70c850e039": {"quality": 0.5318, "cost": 1.95e-06, "time": 0.020999999999999998}, "7112a7e64c": {"quality": 0.64505, "cost": 2.51e-06, "time": 0.0237}, "71b615468b": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027700000000000002}, "723fd5589a": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "7250da0f41": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027700000000000002}, "7274a50778": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "72d022ce33": {"quality": 0.6393, "cost": 2.695e-05, "time": 0.0368}, "7347cf0308": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "736e652158": {"quality": 0.46785, "cost": 1.8299999999999998e-06, "time": 0.020900000000000002}, "739b1f81dc": {"quality": 0.5367, "cost": 1.11e-06, "time": 0.0157}, "742a5c0552": {"quality": 0.7030000000000001, "cost": 1.426e-05, "time": 0.0218}, "742ec1b2e1": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.028900000000000002}, "7435fd54f8": {"quality": 0.5656749999999999, "cost": 1.397e-05, "time": 0.0295}, "7445d99939": {"quality": 0.43596666666666667, "cost": 2.76e-06, "time": 0.0283}, "7466a5f424": {"quality": 0.537525, "cost": 5.0800000000000005e-06, "time": 0.0416}, "74a0be215b": {"quality": 0.5161250000000001, "cost": 1.4420000000000001e-05, "time": 0.0309}, "74d7f64b8c": {"quality": 0.46785, "cost": 1.8299999999999998e-06, "time": 0.020900000000000002}, "7524905580": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "752d9649f2": {"quality": 0.5484, "cost": 5.0800000000000005e-06, "time": 0.044899999999999995}, "75d61c2cd0": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.028900000000000002}, "7604c0aa13": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027699999999999995}, "76c09db721": {"quality": 0.61985, "cost": 1.537e-05, "time": 0.0375}, "774f268b66": {"quality": 0.6417499999999999, "cost": 2.611e-05, "time": 0.0315}, "7765576286": {"quality": 0.4847, "cost": 3.51e-06, "time": 0.0381}, "77c02b00c1": {"quality": 0.48335, "cost": 3.68e-06, "time": 0.033600000000000005}, "7801da66b9": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "782d52674e": {"quality": 0.55235, "cost": 1.481e-05, "time": 0.0315}, "786e5d0af5": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "7878563d63": {"quality": 0.7030000000000001, "cost": 1.426e-05, "time": 0.0218}, "79e1ca9b3c": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.028900000000000002}, "79fad58f07": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "7a207b42a8": {"quality": 0.4183, "cost": 2.2799999999999998e-06, "time": 0.0223}, "7a2cdc546c": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.0289}, "7a42a77788": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.0217}, "7a58d3472b": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "7b3937c1f1": {"quality": 0.7030000000000001, "cost": 1.426e-05, "time": 0.0218}, "7b6f44618e": {"quality": 0.4917750000000001, "cost": 4.52e-06, "time": 0.0422}, "7c62576527": {"quality": 0.561875, "cost": 1.498e-05, "time": 0.0303}, "7c89a2b69e": {"quality": 0.53045, "cost": 4.07e-06, "time": 0.0375}, "7c96c9712f": {"quality": 0.58975, "cost": 1.3700000000000001e-05, "time": 0.0191}, "7ca066aa1c": {"quality": 0.5548, "cost": 1.397e-05, "time": 0.0262}, "7cb5591f27": {"quality": 0.6421, "cost": 2.62e-05, "time": 0.027}, "7cf56a7fbc": {"quality": 0.6129000000000001, "cost": 1.546e-05, "time": 0.033}, "7d44f0959d": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "7d60c38c5c": {"quality": 0.6309, "cost": 1.5e-06, "time": 0.0196}, "7d9b4535ac": {"quality": 0.612775, "cost": 1.436e-05, "time": 0.0334}, "7daf7ff182": {"quality": 0.5082000000000001, "cost": 4.16e-06, "time": 0.0363}, "7dcedb3d02": {"quality": 0.61985, "cost": 1.537e-05, "time": 0.037500000000000006}, "7e22f12cd1": {"quality": 0.5548, "cost": 1.397e-05, "time": 0.0262}, "7e53a50b13": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "7ed07ad40a": {"quality": 0.5648333333333334, "cost": 2.7e-06, "time": 0.030799999999999998}, "7fa67a7656": {"quality": 0.5114666666666666, "cost": 3.32e-06, "time": 0.031}, "7fc6c84bdf": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "7ff8a779cc": {"quality": 0.6067333333333332, "cost": 1.361e-05, "time": 0.0236}, "801af99400": {"quality": 0.6695333333333333, "cost": 1.4e-05, "time": 0.0275}, "806881adcb": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "80ad4122e4": {"quality": 0.5618749999999999, "cost": 1.498e-05, "time": 0.0303}, "80be7df955": {"quality": 0.42074999999999996, "cost": 7.2e-07, "time": 0.0085}, "80bf60c422": {"quality": 0.51495, "cost": 1.11e-06, "time": 0.0124}, "81333c7a33": {"quality": 0.6592, "cost": 5.28e-06, "time": 0.0417}, "813e75210b": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "81660ae8b2": {"quality": 0.494375, "cost": 1.4420000000000001e-05, "time": 0.024300000000000002}, "816958b5d1": {"quality": 0.608975, "cost": 1.537e-05, "time": 0.034199999999999994}, "81ab2ef3f4": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.0164}, "829df73946": {"quality": 0.64505, "cost": 2.51e-06, "time": 0.0237}, "82ea1bd1b9": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "8357183895": {"quality": 0.46540000000000004, "cost": 2.67e-06, "time": 0.0262}, "8392a6083a": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.033600000000000005}, "83aee532b7": {"quality": 0.543925, "cost": 1.397e-05, "time": 0.022899999999999997}, "83b26646c3": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "83c9e66ec6": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "847a0e5db5": {"quality": 0.6129, "cost": 1.546e-05, "time": 0.033}, "847fd49235": {"quality": 0.42799999999999994, "cost": 1.08e-06, "time": 0.0144}, "849100224d": {"quality": 0.5837749999999999, "cost": 2.572e-05, "time": 0.024300000000000002}, "84b91c37ab": {"quality": 0.608975, "cost": 1.537e-05, "time": 0.034199999999999994}, "8519bef585": {"quality": 0.42074999999999996, "cost": 7.2e-07, "time": 0.0085}, "85c94a5505": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "85eda38404": {"quality": 0.58975, "cost": 1.3700000000000001e-05, "time": 0.0191}, "862183bfb9": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "8631e49c94": {"quality": 0.5548, "cost": 1.397e-05, "time": 0.026199999999999998}, "8668f65f05": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "86bf6375af": {"quality": 0.560775, "cost": 1.565e-05, "time": 0.040100000000000004}, "870e2f87b4": {"quality": 0.587075, "cost": 4.63e-06, "time": 0.0402}, "887ad124e1": {"quality": 0.5053, "cost": 1.47e-06, "time": 0.021599999999999998}, "8886cb3082": {"quality": 0.5328999999999999, "cost": 3.23e-06, "time": 0.0322}, "88e71efa9b": {"quality": 0.5594250000000001, "cost": 1.582e-05, "time": 0.03560000000000001}, "8941621423": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "8950a6efe0": {"quality": 0.5439333333333334, "cost": 1.322e-05, "time": 0.0197}, "8961e4d901": {"quality": 0.4102333333333333, "cost": 1.92e-06, "time": 0.016399999999999998}, "8974aa89a0": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "89836d2020": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "89a289907e": {"quality": 0.590875, "cost": 3.62e-06, "time": 0.0394}, "89a35a09b1": {"quality": 0.48714999999999997, "cost": 2.67e-06, "time": 0.0328}, "8a3a35c762": {"quality": 0.626925, "cost": 1.6380000000000002e-05, "time": 0.0416}, "8a50695d1f": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.0217}, "8ab351aa13": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.024399999999999998}, "8ac8b5773a": {"quality": 0.563225, "cost": 1.481e-05, "time": 0.034800000000000005}, "8acd758b7f": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "8b10891ea5": {"quality": 0.5678500000000001, "cost": 1.666e-05, "time": 0.0442}, "8b721bbc6f": {"quality": 0.48714999999999997, "cost": 2.67e-06, "time": 0.0328}, "8b77535cce": {"quality": 0.6174, "cost": 1.6210000000000002e-05, "time": 0.042800000000000005}, "8bbbe0f52a": {"quality": 0.4392333333333333, "cost": 1.92e-06, "time": 0.023}, "8bc184f385": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.026899999999999997}, "8bf5c3eadc": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "8bf80a50cb": {"quality": 0.61985, "cost": 1.537e-05, "time": 0.0375}, "8c274ca255": {"quality": 0.6740250000000001, "cost": 1.677e-05, "time": 0.0455}, "8d7594020b": {"quality": 0.5406666666666667, "cost": 1.4060000000000001e-05, "time": 0.025}, "8d79e03266": {"quality": 0.6497666666666667, "cost": 4.27e-06, "time": 0.037599999999999995}, "8d90814b94": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "8e1a01da19": {"quality": 0.5548, "cost": 1.397e-05, "time": 0.0262}, "8e2498635d": {"quality": 0.7176, "cost": 2.676e-05, "time": 0.0297}, "8e5842ccbd": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.026899999999999997}, "8e5daf241e": {"quality": 0.6695333333333333, "cost": 1.4e-05, "time": 0.0275}, "8e9715ee01": {"quality": 0.6453666666666668, "cost": 2.536e-05, "time": 0.0217}, "8e9b7300d4": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "8f29fab8ac": {"quality": 0.513675, "cost": 1.526e-05, "time": 0.0362}, "8f44d89429": {"quality": 0.6884, "cost": 1.602e-05, "time": 0.035699999999999996}, "8f4caddfe6": {"quality": 0.5869666666666667, "cost": 3.88e-06, "time": 0.033699999999999994}, "8f4edde3f0": {"quality": 0.4857333333333333, "cost": 2.48e-06, "time": 0.0191}, "8f9cefbc22": {"quality": 0.7030000000000001, "cost": 1.426e-05, "time": 0.0218}, "9025e2480f": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "9028588af4": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "9059fd80ad": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "90d5e40c1b": {"quality": 0.42146666666666666, "cost": 2.76e-06, "time": 0.025}, "90d9a86a2a": {"quality": 0.49876666666666675, "cost": 3.15e-06, "time": 0.0322}, "90ff13783c": {"quality": 0.5304500000000001, "cost": 4.07e-06, "time": 0.0375}, "90ff8eb055": {"quality": 0.426725, "cost": 3.1199999999999998e-06, "time": 0.030900000000000004}, "9104e31369": {"quality": 0.53045, "cost": 4.07e-06, "time": 0.0375}, "918983323f": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "91928dfdd9": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.028900000000000002}, "91c800af6b": {"quality": 0.47382500000000005, "cost": 3.51e-06, "time": 0.0348}, "91e841cfd5": {"quality": 0.61985, "cost": 1.537e-05, "time": 0.037500000000000006}, "9253901a1f": {"quality": 0.5594250000000001, "cost": 1.582e-05, "time": 0.0356}, "9288642e53": {"quality": 0.5291, "cost": 2.12e-06, "time": 0.0165}, "92ba9c5be3": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "92c9dcd43b": {"quality": 0.543775, "cost": 3.23e-06, "time": 0.0355}, "93011c0821": {"quality": 0.53045, "cost": 4.07e-06, "time": 0.0375}, "933b4d17dd": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "9373267bdb": {"quality": 0.61985, "cost": 1.537e-05, "time": 0.0375}, "94010928c6": {"quality": 0.42799999999999994, "cost": 1.08e-06, "time": 0.0144}, "9403809e44": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "940c88ddc5": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.0164}, "948f4081ba": {"quality": 0.563225, "cost": 1.481e-05, "time": 0.0348}, "94ac356663": {"quality": 0.5114666666666666, "cost": 3.32e-06, "time": 0.031}, "94dff9a424": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "9508356a2e": {"quality": 0.4763, "cost": 1.47e-06, "time": 0.015}, "9539d0e28c": {"quality": 0.561875, "cost": 1.498e-05, "time": 0.0303}, "956bdcc254": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.0224}, "957d0dafc5": {"quality": 0.6884, "cost": 1.602e-05, "time": 0.035699999999999996}, "9594b0c783": {"quality": 0.5053, "cost": 1.47e-06, "time": 0.021599999999999998}, "95a7b80c2a": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.0289}, "964c671f18": {"quality": 0.587075, "cost": 4.63e-06, "time": 0.0402}, "9679fe2b69": {"quality": 0.5053, "cost": 1.47e-06, "time": 0.021599999999999998}, "968fc95038": {"quality": 0.5053, "cost": 1.47e-06, "time": 0.0216}, "96b487c724": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "96c30205f5": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.024399999999999998}, "96f87d6483": {"quality": 0.53045, "cost": 4.07e-06, "time": 0.0375}, "972c83b002": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.023599999999999996}, "977a4d6b6b": {"quality": 0.5484, "cost": 5.0800000000000005e-06, "time": 0.044899999999999995}, "97bc30bd83": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "980db5f95f": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.0164}, "9836765d41": {"quality": 0.5406666666666666, "cost": 1.4060000000000001e-05, "time": 0.024999999999999998}, "99569e3937": {"quality": 0.6174, "cost": 1.6210000000000002e-05, "time": 0.042800000000000005}, "99ea16a9a6": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "9a5b39370f": {"quality": 0.5509999999999999, "cost": 1.498e-05, "time": 0.026999999999999996}, "9aa4abfb50": {"quality": 0.39899999999999997, "cost": 7.2e-07, "time": 0.0052}, "9ad7a98c31": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "9b3fb79bcb": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "9b6d4915f3": {"quality": 0.43270000000000003, "cost": 3.6e-06, "time": 0.0336}, "9bae5bafc1": {"quality": 0.5869666666666667, "cost": 3.88e-06, "time": 0.033699999999999994}, "9be8a5f317": {"quality": 0.5703, "cost": 1.582e-05, "time": 0.038900000000000004}, "9c549db0a7": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "9c85f8cfcb": {"quality": 0.51495, "cost": 1.11e-06, "time": 0.0124}, "9c8cc46e6c": {"quality": 0.51495, "cost": 1.11e-06, "time": 0.0124}, "9c97d35a30": {"quality": 0.5680999999999999, "cost": 1.86e-06, "time": 0.0255}, "9cbe7858a2": {"quality": 0.50525, "cost": 1.4420000000000001e-05, "time": 0.0276}, "9ce2c3fd98": {"quality": 0.4135, "cost": 1.08e-06, "time": 0.0111}, "9d18cd0737": {"quality": 0.5318, "cost": 1.95e-06, "time": 0.020999999999999998}, "9e06360bc9": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "9f07a95e69": {"quality": 0.61605, "cost": 1.6380000000000002e-05, "time": 0.0383}, "9fb157be35": {"quality": 0.48573333333333335, "cost": 2.48e-06, "time": 0.0191}, "a04ac8e33a": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.0217}, "a04bc6e116": {"quality": 0.5406666666666667, "cost": 1.4060000000000001e-05, "time": 0.025}, "a0b81be5b4": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "a0c85d260e": {"quality": 0.5955, "cost": 5.47e-06, "time": 0.048799999999999996}, "a0dc9f50ac": {"quality": 0.43596666666666667, "cost": 2.76e-06, "time": 0.0283}, "a18225b7b5": {"quality": 0.6884, "cost": 1.602e-05, "time": 0.035699999999999996}, "a1d822289e": {"quality": 0.6421, "cost": 2.62e-05, "time": 0.027}, "a25596c056": {"quality": 0.513675, "cost": 1.526e-05, "time": 0.0362}, "a2811c7324": {"quality": 0.6067333333333332, "cost": 1.361e-05, "time": 0.0236}, "a2aa082d14": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027700000000000002}, "a2cd339ad9": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.040799999999999996}, "a2fd03e6a5": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "a31e87d7cb": {"quality": 0.4425, "cost": 7.2e-07, "time": 0.0118}, "a3e23c327b": {"quality": 0.561875, "cost": 1.498e-05, "time": 0.030299999999999997}, "a457f6c300": {"quality": 0.4135, "cost": 1.08e-06, "time": 0.0111}, "a47de025c8": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "a515a9c8cc": {"quality": 0.4376, "cost": 1.5599999999999999e-06, "time": 0.0171}, "a5949b76ec": {"quality": 0.42146666666666666, "cost": 2.76e-06, "time": 0.025}, "a60dd076b8": {"quality": 0.4392333333333333, "cost": 1.92e-06, "time": 0.023}, "a6297a6c56": {"quality": 0.4376, "cost": 1.5599999999999999e-06, "time": 0.0171}, "a62b7555b9": {"quality": 0.5703, "cost": 1.582e-05, "time": 0.0389}, "a63f48e8ca": {"quality": 0.5406666666666667, "cost": 1.4060000000000001e-05, "time": 0.025}, "a6460dbb7c": {"quality": 0.64505, "cost": 2.51e-06, "time": 0.0237}, "a66a4cf4b0": {"quality": 0.61605, "cost": 1.6380000000000002e-05, "time": 0.0383}, "a6e2d69222": {"quality": 0.6129000000000001, "cost": 1.546e-05, "time": 0.033}, "a717c4c535": {"quality": 0.4917750000000001, "cost": 4.52e-06, "time": 0.0422}, "a76afe9960": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "a7a6353090": {"quality": 0.6067333333333332, "cost": 1.361e-05, "time": 0.0236}, "a80f6535b1": {"quality": 0.59465, "cost": 1.286e-05, "time": 0.0138}, "a86b137d7f": {"quality": 0.5508500000000001, "cost": 2.12e-06, "time": 0.019799999999999998}, "a88eb1493c": {"quality": 0.5318, "cost": 1.95e-06, "time": 0.020999999999999998}, "a89c533d6c": {"quality": 0.5374, "cost": 1.4900000000000001e-05, "time": 0.0303}, "a8d8264600": {"quality": 0.6174000000000001, "cost": 1.621e-05, "time": 0.042800000000000005}, "a8eb36b210": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.0289}, "a95b4a6dd0": {"quality": 0.42146666666666666, "cost": 2.76e-06, "time": 0.025}, "a9621ea4e6": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.0197}, "a9721a0a50": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.030999999999999996}, "a972b02c61": {"quality": 0.7468, "cost": 1.25e-05, "time": 0.0079}, "a9c5c4e311": {"quality": 0.6067333333333332, "cost": 1.361e-05, "time": 0.0236}, "a9d96670eb": {"quality": 0.5439333333333334, "cost": 1.322e-05, "time": 0.0197}, "a9e8c974d3": {"quality": 0.5509999999999999, "cost": 1.498e-05, "time": 0.026999999999999996}, "aa08180e36": {"quality": 0.4858, "cost": 2.84e-06, "time": 0.0283}, "aa38702a02": {"quality": 0.42799999999999994, "cost": 1.08e-06, "time": 0.0144}, "aadbfc418b": {"quality": 0.44250000000000006, "cost": 1.08e-06, "time": 0.0177}, "aaeb8b0010": {"quality": 0.50525, "cost": 1.4420000000000001e-05, "time": 0.0276}, "ab288ee7f2": {"quality": 0.563225, "cost": 1.481e-05, "time": 0.0348}, "ab43b02cb0": {"quality": 0.5082000000000001, "cost": 4.16e-06, "time": 0.0363}, "aba1d612cc": {"quality": 0.648825, "cost": 2.712e-05, "time": 0.0356}, "ac208e7a1d": {"quality": 0.5724666666666667, "cost": 3.88e-06, "time": 0.0304}, "ac2224adbe": {"quality": 0.6129000000000001, "cost": 1.546e-05, "time": 0.033}, "ac828ffe70": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "ac9fdc1550": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "aca957ecff": {"quality": 0.6417499999999999, "cost": 2.611e-05, "time": 0.0315}, "acfe1ed920": {"quality": 0.59465, "cost": 1.286e-05, "time": 0.0138}, "ad3efe44c3": {"quality": 0.39899999999999997, "cost": 7.2e-07, "time": 0.0052}, "ad41c95a99": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "ad48432c22": {"quality": 0.46540000000000004, "cost": 2.67e-06, "time": 0.0262}, "ad6ebbba8d": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "ad90055ef6": {"quality": 0.39899999999999997, "cost": 7.2e-07, "time": 0.0052}, "adab1e0fb1": {"quality": 0.5508500000000001, "cost": 2.12e-06, "time": 0.019799999999999998}, "ae655ec593": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "ae94b172be": {"quality": 0.5002333333333333, "cost": 2.48e-06, "time": 0.0224}, "aec9dc5873": {"quality": 0.5729, "cost": 1.286e-05, "time": 0.0105}, "af360c323c": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.018299999999999997}, "af90567194": {"quality": 0.587075, "cost": 4.63e-06, "time": 0.0402}, "afe77d0f89": {"quality": 0.66695, "cost": 1.576e-05, "time": 0.041400000000000006}, "b0948c05b6": {"quality": 0.5837, "cost": 4.7200000000000005e-06, "time": 0.03899999999999999}, "b0c4a6640b": {"quality": 0.5548, "cost": 1.397e-05, "time": 0.0262}, "b12caafd58": {"quality": 0.5076999999999999, "cost": 1.358e-05, "time": 0.0223}, "b18168b9c1": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "b1a7428a01": {"quality": 0.5703, "cost": 1.582e-05, "time": 0.038900000000000004}, "b1acdebb48": {"quality": 0.6884, "cost": 1.602e-05, "time": 0.035699999999999996}, "b1b06f4ee7": {"quality": 0.6421, "cost": 2.62e-05, "time": 0.027000000000000003}, "b1cf8d33e5": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "b1e9ab6b1a": {"quality": 0.559425, "cost": 1.582e-05, "time": 0.0356}, "b2b057ba41": {"quality": 0.5680999999999999, "cost": 1.86e-06, "time": 0.0255}, "b2e063499d": {"quality": 0.6497666666666667, "cost": 4.27e-06, "time": 0.037599999999999995}, "b33412410e": {"quality": 0.5374, "cost": 1.4900000000000001e-05, "time": 0.0303}, "b3369775dc": {"quality": 0.588425, "cost": 4.46e-06, "time": 0.044700000000000004}, "b3b9205f60": {"quality": 0.50525, "cost": 1.4420000000000001e-05, "time": 0.027600000000000003}, "b3c56f0b3c": {"quality": 0.6421, "cost": 2.62e-05, "time": 0.027}, "b3f20b706d": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "b4002173ee": {"quality": 0.43270000000000003, "cost": 2.4e-06, "time": 0.0224}, "b46d382384": {"quality": 0.6453666666666668, "cost": 2.536e-05, "time": 0.0217}, "b4b2482ef9": {"quality": 0.6129000000000001, "cost": 1.546e-05, "time": 0.033}, "b4be043238": {"quality": 0.6308666666666666, "cost": 2.536e-05, "time": 0.0184}, "b531bd0548": {"quality": 0.581325, "cost": 2.6560000000000003e-05, "time": 0.0296}, "b56c312eda": {"quality": 0.4847, "cost": 3.51e-06, "time": 0.0381}, "b5e2b41c1c": {"quality": 0.5536, "cost": 1.86e-06, "time": 0.022199999999999998}, "b61ce57a90": {"quality": 0.5703, "cost": 1.582e-05, "time": 0.038900000000000004}, "b64ddb14f9": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "b67107a43e": {"quality": 0.46785, "cost": 1.8299999999999998e-06, "time": 0.020900000000000002}, "b67720aa5c": {"quality": 0.5149333333333334, "cost": 1.322e-05, "time": 0.0131}, "b682a23b89": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "b69ef5add4": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.0164}, "b796b7ffd3": {"quality": 0.590875, "cost": 3.62e-06, "time": 0.0394}, "b7a0083dc4": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "b7d0e8557f": {"quality": 0.48714999999999997, "cost": 2.67e-06, "time": 0.0328}, "b8317a3a8c": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "b8ab3d2f25": {"quality": 0.5869666666666667, "cost": 3.88e-06, "time": 0.033699999999999994}, "b8b569172f": {"quality": 0.4969666666666666, "cost": 3.32e-06, "time": 0.0277}, "b8f5ab44bb": {"quality": 0.6244750000000001, "cost": 1.722e-05, "time": 0.0469}, "b91e7fdb29": {"quality": 0.4183, "cost": 2.2799999999999998e-06, "time": 0.0223}, "b932beaaa6": {"quality": 0.6129, "cost": 1.546e-05, "time": 0.033}, "b9770c2261": {"quality": 0.6309, "cost": 1.5e-06, "time": 0.0196}, "b9bb1e6f8d": {"quality": 0.588425, "cost": 4.46e-06, "time": 0.044700000000000004}, "b9d0e8740c": {"quality": 0.5703, "cost": 1.582e-05, "time": 0.038900000000000004}, "b9da208432": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "ba3223f6ac": {"quality": 0.428, "cost": 1.08e-06, "time": 0.0144}, "bb13365175": {"quality": 0.608975, "cost": 1.537e-05, "time": 0.034199999999999994}, "bb1b3a4d29": {"quality": 0.6019, "cost": 1.436e-05, "time": 0.0301}, "bb6536b0ab": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.030999999999999996}, "bbba9dd6ae": {"quality": 0.64505, "cost": 2.51e-06, "time": 0.0237}, "bbde69a1ae": {"quality": 0.5028, "cost": 1.526e-05, "time": 0.0329}, "bc29a0c0fe": {"quality": 0.6453666666666668, "cost": 2.536e-05, "time": 0.0217}, "bc3d02f753": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "bc4c1fcc64": {"quality": 0.64505, "cost": 2.51e-06, "time": 0.0237}, "bd30d27f62": {"quality": 0.6308666666666666, "cost": 2.536e-05, "time": 0.0184}, "bd99b2fb21": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.031}, "bddc7d2a34": {"quality": 0.6019, "cost": 1.436e-05, "time": 0.0301}, "be2ae88f70": {"quality": 0.4135, "cost": 1.08e-06, "time": 0.0111}, "be4740f38f": {"quality": 0.5837, "cost": 4.7200000000000005e-06, "time": 0.03899999999999999}, "bec0c6a95f": {"quality": 0.581325, "cost": 2.6560000000000003e-05, "time": 0.0296}, "bed888d4dc": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "bf45e407f6": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "bf5550f320": {"quality": 0.5703, "cost": 1.582e-05, "time": 0.0389}, "bf87e58322": {"quality": 0.6695333333333333, "cost": 1.4e-05, "time": 0.0275}, "bfed7670ed": {"quality": 0.4969666666666666, "cost": 3.32e-06, "time": 0.0277}, "c0541e2220": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.0217}, "c0e10c0048": {"quality": 0.50525, "cost": 1.4420000000000001e-05, "time": 0.0276}, "c127509a7a": {"quality": 0.6403333333333333, "cost": 3.26e-06, "time": 0.0335}, "c13682c7c7": {"quality": 0.494225, "cost": 3.68e-06, "time": 0.0369}, "c13d6e78e9": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "c14ff3144d": {"quality": 0.4425, "cost": 7.2e-07, "time": 0.0118}, "c1e42ac47b": {"quality": 0.6244750000000001, "cost": 1.722e-05, "time": 0.0469}, "c2949aa902": {"quality": 0.5648333333333334, "cost": 2.7e-06, "time": 0.030799999999999998}, "c31e956b35": {"quality": 0.523375, "cost": 3.06e-06, "time": 0.0334}, "c36b525dde": {"quality": 0.5648333333333334, "cost": 2.7e-06, "time": 0.030799999999999998}, "c38326e2bd": {"quality": 0.626925, "cost": 1.6380000000000002e-05, "time": 0.0416}, "c3ec2cec59": {"quality": 0.5724666666666667, "cost": 3.88e-06, "time": 0.0304}, "c44720575f": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "c48ecefab6": {"quality": 0.49682499999999996, "cost": 1.358e-05, "time": 0.019000000000000003}, "c4a64eb40f": {"quality": 0.7176, "cost": 2.676e-05, "time": 0.0297}, "c4a80d19b3": {"quality": 0.5800000000000001, "cost": 3.62e-06, "time": 0.03609999999999999}, "c4c2826afd": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "c4c94a5527": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "c4e75ee9ba": {"quality": 0.7176, "cost": 2.676e-05, "time": 0.0297}, "c4f3e7665d": {"quality": 0.4135, "cost": 1.08e-06, "time": 0.0111}, "c5471bef57": {"quality": 0.494375, "cost": 1.4420000000000001e-05, "time": 0.024300000000000002}, "c54a408db7": {"quality": 0.6067333333333333, "cost": 1.361e-05, "time": 0.0236}, "c59cc41335": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "c5a0b065e0": {"quality": 0.5729, "cost": 1.286e-05, "time": 0.0105}, "c5a16b834a": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "c5fbe2076f": {"quality": 0.537525, "cost": 5.0800000000000005e-06, "time": 0.0416}, "c617370f6b": {"quality": 0.5439333333333334, "cost": 1.322e-05, "time": 0.0197}, "c67f782c7f": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.024399999999999998}, "c691a29c42": {"quality": 0.48090000000000005, "cost": 4.52e-06, "time": 0.038900000000000004}, "c6a339987c": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "c772ff3704": {"quality": 0.429175, "cost": 2.2799999999999998e-06, "time": 0.0256}, "c7e3f348c2": {"quality": 0.61605, "cost": 1.6380000000000002e-05, "time": 0.0383}, "c823589ab6": {"quality": 0.5149333333333334, "cost": 1.322e-05, "time": 0.0131}, "c82f834e85": {"quality": 0.5775333333333333, "cost": 2.87e-06, "time": 0.029599999999999998}, "c85099881f": {"quality": 0.5729, "cost": 1.286e-05, "time": 0.0105}, "c935a33384": {"quality": 0.5484, "cost": 5.0800000000000005e-06, "time": 0.044899999999999995}, "ca3177461f": {"quality": 0.50525, "cost": 1.4420000000000001e-05, "time": 0.0276}, "caa7c0bd6b": {"quality": 0.49876666666666675, "cost": 3.15e-06, "time": 0.0322}, "cac6b051e9": {"quality": 0.53425, "cost": 3.06e-06, "time": 0.036699999999999997}, "cacb342f64": {"quality": 0.63795, "cost": 2.712e-05, "time": 0.032299999999999995}, "cb9948679c": {"quality": 0.5076999999999999, "cost": 1.358e-05, "time": 0.0223}, "cbb5eb0e74": {"quality": 0.53425, "cost": 3.06e-06, "time": 0.036699999999999997}, "cbc32cbeff": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.029500000000000002}, "cbd4461293": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "cbe2318045": {"quality": 0.5147333333333334, "cost": 2.48e-06, "time": 0.025699999999999997}, "cc886fe337": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "cc9a6248a0": {"quality": 0.7468, "cost": 1.25e-05, "time": 0.0079}, "ccb2335b3f": {"quality": 0.55235, "cost": 1.481e-05, "time": 0.0315}, "ccdf03a55b": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.0164}, "ccf72745c1": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027700000000000002}, "cd1d418732": {"quality": 0.5082000000000001, "cost": 4.16e-06, "time": 0.0363}, "cd23c79db1": {"quality": 0.48573333333333335, "cost": 2.48e-06, "time": 0.0191}, "cd64fbfcd9": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.023599999999999996}, "cd85a01e81": {"quality": 0.55235, "cost": 1.481e-05, "time": 0.0315}, "ce4bc5f348": {"quality": 0.42074999999999996, "cost": 7.2e-07, "time": 0.0085}, "ce980cf86f": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "ceae8b8bb9": {"quality": 0.608975, "cost": 1.537e-05, "time": 0.034199999999999994}, "cecca90dd2": {"quality": 0.5536, "cost": 1.86e-06, "time": 0.022199999999999998}, "cf9538faf0": {"quality": 0.5147333333333334, "cost": 2.48e-06, "time": 0.0257}, "cf9d2e224c": {"quality": 0.5508500000000001, "cost": 2.12e-06, "time": 0.019799999999999998}, "cfd36f3a8c": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.0263}, "cffa29a6ef": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "d03596c3de": {"quality": 0.5328999999999999, "cost": 3.23e-06, "time": 0.0322}, "d07b766487": {"quality": 0.57275, "cost": 1.498e-05, "time": 0.0336}, "d0a0a66d75": {"quality": 0.6129000000000001, "cost": 1.546e-05, "time": 0.033}, "d0ce31134c": {"quality": 0.5594250000000001, "cost": 1.582e-05, "time": 0.0356}, "d0f9633442": {"quality": 0.4102333333333333, "cost": 1.92e-06, "time": 0.016399999999999998}, "d216eab7d8": {"quality": 0.5869666666666667, "cost": 3.88e-06, "time": 0.033699999999999994}, "d266c19ac8": {"quality": 0.525825, "cost": 2.22e-06, "time": 0.0281}, "d26a70179a": {"quality": 0.608975, "cost": 1.537e-05, "time": 0.034199999999999994}, "d2af24b59e": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.0224}, "d2f2dd5cd4": {"quality": 0.5439333333333334, "cost": 1.322e-05, "time": 0.0197}, "d302278f85": {"quality": 0.5548, "cost": 1.397e-05, "time": 0.0262}, "d37dcaea30": {"quality": 0.6174000000000001, "cost": 1.6210000000000002e-05, "time": 0.042800000000000005}, "d3a2d50bd7": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.0183}, "d3d4185487": {"quality": 0.6016666666666667, "cost": 1.462e-05, "time": 0.024399999999999998}, "d3db4cf84d": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "d402233b53": {"quality": 0.608975, "cost": 1.537e-05, "time": 0.034199999999999994}, "d43fafa19e": {"quality": 0.41585, "cost": 1.5599999999999999e-06, "time": 0.0138}, "d446a75eb7": {"quality": 0.7176, "cost": 2.676e-05, "time": 0.0297}, "d48ead13da": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.022399999999999996}, "d5016f4538": {"quality": 0.41585, "cost": 1.5599999999999999e-06, "time": 0.0138}, "d55a3613b0": {"quality": 0.695925, "cost": 2.751e-05, "time": 0.03950000000000001}, "d58036ba66": {"quality": 0.5002333333333334, "cost": 2.48e-06, "time": 0.0224}, "d5a84c782e": {"quality": 0.6695333333333333, "cost": 1.4e-05, "time": 0.0275}, "d5b2eef11c": {"quality": 0.68885, "cost": 1.325e-05, "time": 0.0177}, "d6040140b9": {"quality": 0.5955, "cost": 5.47e-06, "time": 0.048799999999999996}, "d65185c1a4": {"quality": 0.5680999999999999, "cost": 1.86e-06, "time": 0.0255}, "d667351f33": {"quality": 0.6174000000000001, "cost": 1.621e-05, "time": 0.042800000000000005}, "d6bd3b66ba": {"quality": 0.4135, "cost": 1.08e-06, "time": 0.0111}, "d6c4e48eeb": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.0263}, "d6cbf265ee": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "d705447fd7": {"quality": 0.608975, "cost": 1.537e-05, "time": 0.0342}, "d73a9aab4e": {"quality": 0.41585, "cost": 1.5599999999999999e-06, "time": 0.0138}, "d752c30d07": {"quality": 0.5002333333333333, "cost": 2.48e-06, "time": 0.0224}, "d782682359": {"quality": 0.5618749999999999, "cost": 1.498e-05, "time": 0.0303}, "d7c0972014": {"quality": 0.5082, "cost": 4.16e-06, "time": 0.0363}, "d867525748": {"quality": 0.5406666666666666, "cost": 1.4060000000000001e-05, "time": 0.025}, "d87eb775da": {"quality": 0.6403333333333333, "cost": 3.26e-06, "time": 0.0335}, "d8bab6c09b": {"quality": 0.53045, "cost": 4.07e-06, "time": 0.0375}, "d8bcac36e8": {"quality": 0.4969666666666667, "cost": 3.32e-06, "time": 0.027699999999999995}, "d8eadc0190": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "d96677d8d4": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.026899999999999997}, "d98f22270e": {"quality": 0.6129000000000001, "cost": 1.546e-05, "time": 0.033}, "d9e2bb21a3": {"quality": 0.5053, "cost": 1.47e-06, "time": 0.021599999999999998}, "da95deeb20": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.0289}, "daaadadcc9": {"quality": 0.4858, "cost": 2.84e-06, "time": 0.0283}, "daf855e065": {"quality": 0.525825, "cost": 2.22e-06, "time": 0.0281}, "db00594832": {"quality": 0.6129, "cost": 1.546e-05, "time": 0.033}, "db19e677c4": {"quality": 0.5149333333333334, "cost": 1.322e-05, "time": 0.0131}, "db3c035639": {"quality": 0.626925, "cost": 1.6380000000000002e-05, "time": 0.0416}, "db41487005": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.016399999999999998}, "db6a7482fd": {"quality": 0.68885, "cost": 1.325e-05, "time": 0.0177}, "db9060cd27": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "dbce95a072": {"quality": 0.5114666666666666, "cost": 3.32e-06, "time": 0.031}, "dc195abe5e": {"quality": 0.5548, "cost": 1.397e-05, "time": 0.0262}, "dc3f4b7138": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.016399999999999998}, "dc66bccb1c": {"quality": 0.5002333333333333, "cost": 2.48e-06, "time": 0.0224}, "dc90065dea": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "dd0d70fedd": {"quality": 0.6309, "cost": 1.5e-06, "time": 0.0196}, "dddc76b3ca": {"quality": 0.5374, "cost": 1.4900000000000001e-05, "time": 0.0303}, "de18bf45e1": {"quality": 0.4425, "cost": 3.6e-07, "time": 0.0059}, "de1e56370f": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "df2160ecc8": {"quality": 0.5374, "cost": 1.4900000000000001e-05, "time": 0.0303}, "dfda94bd2a": {"quality": 0.42074999999999996, "cost": 7.2e-07, "time": 0.0085}, "dff452a9ca": {"quality": 0.4102333333333333, "cost": 1.92e-06, "time": 0.016399999999999998}, "e09f75d9d6": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "e0b6a99753": {"quality": 0.7468, "cost": 1.25e-05, "time": 0.0079}, "e0cf6587a7": {"quality": 0.50525, "cost": 1.4420000000000001e-05, "time": 0.0276}, "e0de4a5929": {"quality": 0.6174000000000001, "cost": 1.6210000000000002e-05, "time": 0.042800000000000005}, "e1356fb426": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.0289}, "e20ba014a1": {"quality": 0.59795, "cost": 4.63e-06, "time": 0.0435}, "e21806e3bc": {"quality": 0.5406666666666666, "cost": 1.4060000000000001e-05, "time": 0.025}, "e24e97564c": {"quality": 0.66695, "cost": 1.576e-05, "time": 0.041400000000000006}, "e2673c1ec8": {"quality": 0.6174, "cost": 1.6210000000000002e-05, "time": 0.042800000000000005}, "e26c7bfbdb": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.0197}, "e2f9980b06": {"quality": 0.6174000000000001, "cost": 1.6210000000000002e-05, "time": 0.042800000000000005}, "e3445f7632": {"quality": 0.68885, "cost": 1.325e-05, "time": 0.0177}, "e35e5f81a7": {"quality": 0.4763, "cost": 1.47e-06, "time": 0.015}, "e376ac53e7": {"quality": 0.5294333333333333, "cost": 1.322e-05, "time": 0.016399999999999998}, "e3d8bb56da": {"quality": 0.6067333333333332, "cost": 1.361e-05, "time": 0.0236}, "e3df4cf041": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "e47dc3abca": {"quality": 0.5082000000000001, "cost": 4.16e-06, "time": 0.0363}, "e4b9d4fb41": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "e510bda989": {"quality": 0.6592, "cost": 1.76e-06, "time": 0.0139}, "e517cd2222": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "e51b01f418": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "e520dfae5b": {"quality": 0.47872499999999996, "cost": 1.8299999999999998e-06, "time": 0.0242}, "e521c9b7e4": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.0217}, "e54097ad5d": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "e56a16ca66": {"quality": 0.5367, "cost": 1.11e-06, "time": 0.0157}, "e5c4abf7ce": {"quality": 0.5632250000000001, "cost": 1.481e-05, "time": 0.0348}, "e5d4689312": {"quality": 0.494375, "cost": 1.4420000000000001e-05, "time": 0.024300000000000002}, "e62a7b27ae": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.021699999999999997}, "e6a7aff3bc": {"quality": 0.543925, "cost": 1.397e-05, "time": 0.022899999999999997}, "e736999157": {"quality": 0.543925, "cost": 1.397e-05, "time": 0.022899999999999997}, "e7517a8ce0": {"quality": 0.7081666666666666, "cost": 2.5750000000000002e-05, "time": 0.0256}, "e7520ca5ac": {"quality": 0.5632250000000001, "cost": 1.481e-05, "time": 0.0348}, "e7e94ab7a5": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "e887ddf5cc": {"quality": 0.648825, "cost": 2.712e-05, "time": 0.0356}, "e94fb5a295": {"quality": 0.5149333333333334, "cost": 1.322e-05, "time": 0.0131}, "ea6ecc5653": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.0236}, "ea8bcb3ae2": {"quality": 0.5837, "cost": 4.7200000000000005e-06, "time": 0.03899999999999999}, "ebbe8b6c4f": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.031}, "ebdf3abff2": {"quality": 0.6497666666666667, "cost": 4.27e-06, "time": 0.037599999999999995}, "ec55dba809": {"quality": 0.561875, "cost": 1.498e-05, "time": 0.030299999999999997}, "ecb5f78f37": {"quality": 0.522025, "cost": 3.23e-06, "time": 0.028899999999999995}, "ecda3d74cb": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.021699999999999997}, "ece10c0388": {"quality": 0.6453666666666668, "cost": 2.536e-05, "time": 0.0217}, "ed6b5480a5": {"quality": 0.4376, "cost": 1.5599999999999999e-06, "time": 0.0171}, "eda630dc85": {"quality": 0.6403333333333333, "cost": 3.26e-06, "time": 0.0335}, "edaaee5ed4": {"quality": 0.54595, "cost": 2.96e-06, "time": 0.025099999999999997}, "edb2b764aa": {"quality": 0.5869666666666666, "cost": 3.88e-06, "time": 0.0337}, "edc52339db": {"quality": 0.47247500000000003, "cost": 3.68e-06, "time": 0.0303}, "ede7071775": {"quality": 0.7081666666666667, "cost": 2.5750000000000002e-05, "time": 0.0256}, "ee46042c5d": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.0336}, "ee68c51f73": {"quality": 0.7468, "cost": 2.5e-05, "time": 0.0158}, "ee7b726747": {"quality": 0.61985, "cost": 1.537e-05, "time": 0.0375}, "eec5f32da9": {"quality": 0.6393, "cost": 2.695e-05, "time": 0.0368}, "eed40e4378": {"quality": 0.61985, "cost": 1.537e-05, "time": 0.0375}, "eef12d478b": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "ef37b3e0be": {"quality": 0.4183, "cost": 2.2799999999999998e-06, "time": 0.0223}, "ef43d497f1": {"quality": 0.48335, "cost": 3.68e-06, "time": 0.033600000000000005}, "ef4d4c4a62": {"quality": 0.5869666666666667, "cost": 3.88e-06, "time": 0.033699999999999994}, "ef9a651425": {"quality": 0.6740250000000001, "cost": 1.677e-05, "time": 0.0455}, "f0655621af": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "f076b4c9ae": {"quality": 0.5630333333333334, "cost": 2.87e-06, "time": 0.026299999999999997}, "f11eddb4ed": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "f1408da253": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.0295}, "f18cf41929": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}, "f1aa0b0b42": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "f1bda127f6": {"quality": 0.5114666666666666, "cost": 3.32e-06, "time": 0.031}, "f1f373e58e": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "f2a2e91541": {"quality": 0.476275, "cost": 2.67e-06, "time": 0.029500000000000002}, "f2c04ed1c8": {"quality": 0.4376, "cost": 1.5599999999999999e-06, "time": 0.0171}, "f2cf5db12d": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.031}, "f366c0dd10": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "f4303a5b4f": {"quality": 0.5922000000000001, "cost": 2.6560000000000003e-05, "time": 0.0329}, "f437481e3b": {"quality": 0.5399750000000001, "cost": 4.24e-06, "time": 0.0363}, "f4bc6b63a7": {"quality": 0.7081666666666666, "cost": 2.5750000000000002e-05, "time": 0.0256}, "f4dc556633": {"quality": 0.608975, "cost": 1.537e-05, "time": 0.0342}, "f4deb72db6": {"quality": 0.4102333333333333, "cost": 1.92e-06, "time": 0.016399999999999998}, "f4ef2b9c33": {"quality": 0.6034666666666667, "cost": 1.445e-05, "time": 0.0289}, "f566d6d6a1": {"quality": 0.6161666666666666, "cost": 1.462e-05, "time": 0.0277}, "f5b9a94dcc": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "f5c27e7172": {"quality": 0.49079999999999996, "cost": 1.47e-06, "time": 0.018299999999999997}, "f5e53d963b": {"quality": 0.4376, "cost": 1.5599999999999999e-06, "time": 0.0171}, "f614235c15": {"quality": 0.39899999999999997, "cost": 3.6e-07, "time": 0.0026}, "f6546149e3": {"quality": 0.7081666666666666, "cost": 2.5750000000000002e-05, "time": 0.0256}, "f74ec023e4": {"quality": 0.494225, "cost": 3.68e-06, "time": 0.0369}, "f7b048bd54": {"quality": 0.5742666666666667, "cost": 3.71e-06, "time": 0.0349}, "f7c4df993e": {"quality": 0.4763, "cost": 1.47e-06, "time": 0.015}, "f854533145": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "f89b8a1930": {"quality": 0.5548, "cost": 1.397e-05, "time": 0.0262}, "f93d9a2693": {"quality": 0.5020333333333333, "cost": 2.31e-06, "time": 0.0269}, "f97d91a249": {"quality": 0.5406666666666667, "cost": 1.4060000000000001e-05, "time": 0.025}, "f99096d89c": {"quality": 0.543775, "cost": 3.23e-06, "time": 0.0355}, "f9e8e221f3": {"quality": 0.43596666666666667, "cost": 2.76e-06, "time": 0.0283}, "fa38879eab": {"quality": 0.538875, "cost": 4.9100000000000004e-06, "time": 0.0461}, "fa71111570": {"quality": 0.55235, "cost": 1.481e-05, "time": 0.0315}, "fa7882d46b": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "fa906520d1": {"quality": 0.5413250000000001, "cost": 4.07e-06, "time": 0.0408}, "faabebaa30": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "fb0339a7d0": {"quality": 0.5304500000000001, "cost": 4.07e-06, "time": 0.0375}, "fb216ad6b3": {"quality": 0.43270000000000003, "cost": 1.2e-06, "time": 0.0112}, "fb6216880a": {"quality": 0.522025, "cost": 3.23e-06, "time": 0.028899999999999995}, "fba499b89d": {"quality": 0.561875, "cost": 1.498e-05, "time": 0.0303}, "fbc010a368": {"quality": 0.5261666666666667, "cost": 1.4060000000000001e-05, "time": 0.0217}, "fbc02e2e07": {"quality": 0.5406666666666666, "cost": 1.4060000000000001e-05, "time": 0.024999999999999998}, "fbd6c45271": {"quality": 0.5329, "cost": 3.23e-06, "time": 0.0322}, "fc0a156e16": {"quality": 0.6789666666666667, "cost": 1.501e-05, "time": 0.0316}, "fc1fd5bf54": {"quality": 0.47872499999999996, "cost": 1.8299999999999998e-06, "time": 0.0242}, "fc6967a75b": {"quality": 0.49682499999999996, "cost": 1.358e-05, "time": 0.019000000000000003}, "fc73c3b0fa": {"quality": 0.7468, "cost": 1.25e-05, "time": 0.0079}, "fce38334b2": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.033600000000000005}, "fce5fca128": {"quality": 0.6174000000000001, "cost": 1.6210000000000002e-05, "time": 0.042800000000000005}, "fd0709359e": {"quality": 0.6309, "cost": 7.5e-07, "time": 0.0098}, "fd1f809d64": {"quality": 0.5922333333333333, "cost": 1.361e-05, "time": 0.0203}, "fd2c994a9d": {"quality": 0.5548, "cost": 1.397e-05, "time": 0.026199999999999998}, "fddccfbf94": {"quality": 0.5114666666666667, "cost": 3.32e-06, "time": 0.030999999999999996}, "fe7fa741b4": {"quality": 0.6129000000000001, "cost": 1.546e-05, "time": 0.033}, "fe9e1fec71": {"quality": 0.6129, "cost": 1.546e-05, "time": 0.033}, "fea4734c09": {"quality": 0.54595, "cost": 2.96e-06, "time": 0.025099999999999997}, "fef1ca27fa": {"quality": 0.4917750000000001, "cost": 4.52e-06, "time": 0.0422}, "ff11cb6a7a": {"quality": 0.48335000000000006, "cost": 3.68e-06, "time": 0.0336}, "ff171e34e2": {"quality": 0.4875333333333334, "cost": 2.31e-06, "time": 0.023599999999999996}, "ff1c958e21": {"quality": 0.51495, "cost": 1.11e-06, "time": 0.0124}, "ff8df4ace9": {"quality": 0.5374, "cost": 1.4900000000000001e-05, "time": 0.0303}, "ff8e68049a": {"quality": 0.42473333333333335, "cost": 1.92e-06, "time": 0.019700000000000002}} ================================================ FILE: abacus-research/cuad-demo.py ================================================ import argparse import json import os import string import numpy as np import pandas as pd from cuad_data_loader import load_cuad_data import palimpzest as pz from palimpzest.constants import Model CUAD_CATEGORIES = [ { "Category": "Document Name", "Description": "The name of the contract", "Answer Format": "Contract Name", "Group": "Group: -", }, { "Category": "Parties", "Description": "The two or more parties who signed the contract", "Answer Format": "Entity or individual names", "Group": "Group: -", }, { "Category": "Agreement Date", "Description": "The date of the contract", "Answer Format": "Date (mm/dd/yyyy)", "Group": "Group: 1", }, { "Category": "Effective Date", "Description": "The date when the contract is effective\u00a0", "Answer Format": "Date (mm/dd/yyyy)", "Group": "Group: 1", }, { "Category": "Expiration Date", "Description": "On what date will the contract's initial term expire?", "Answer Format": "Date (mm/dd/yyyy) / Perpetual", "Group": "Group: 1", }, { "Category": "Renewal Term", "Description": "What is the renewal term after the initial term expires? This includes automatic extensions and unilateral extensions with prior notice.", "Answer Format": "[Successive] number of years/months / Perpetual", "Group": "Group: 1", }, { "Category": "Notice Period to Terminate Renewal", "Description": "What is the notice period required to terminate renewal?", "Answer Format": "Number of days/months/year(s)", "Group": "Group: 1", }, { "Category": "Governing Law", "Description": "Which state/country's law governs the interpretation of the contract?", "Answer Format": "Name of a US State / non-US Province, Country", "Group": "Group: -", }, { "Category": "Most Favored Nation", "Description": "Is there a clause that if a third party gets better terms on the licensing or sale of technology/goods/services described in the contract, the buyer of such technology/goods/services under the contract shall be entitled to those better terms?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Non-Compete", "Description": "Is there a restriction on the ability of a party to compete with the counterparty or operate in a certain geography or business or technology sector?\u00a0", "Answer Format": "Yes/No", "Group": "Group: 2", }, { "Category": "Exclusivity", "Description": "Is there an exclusive dealing\u00a0 commitment with the counterparty? This includes a commitment to procure all \u201crequirements\u201d from one party of certain technology, goods, or services or a prohibition on licensing or selling technology, goods or services to third parties, or a prohibition on\u00a0 collaborating or working with other parties), whether during the contract or\u00a0 after the contract ends (or both).", "Answer Format": "Yes/No", "Group": "Group: 2", }, { "Category": "No-Solicit of Customers", "Description": "Is a party restricted from contracting or soliciting customers or partners of the counterparty, whether during the contract or after the contract ends (or both)?", "Answer Format": "Yes/No", "Group": "Group: 2", }, { "Category": "Competitive Restriction Exception", "Description": "This category includes the exceptions or carveouts to Non-Compete, Exclusivity and No-Solicit of Customers above.", "Answer Format": "Yes/No", "Group": "Group: 2", }, { "Category": "No-Solicit of Employees", "Description": "Is there a restriction on a party\u2019s soliciting or hiring employees and/or contractors from the\u00a0 counterparty, whether during the contract or after the contract ends (or both)?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Non-Disparagement", "Description": "Is there a requirement on a party not to disparage the counterparty?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Termination for Convenience", "Description": "Can a party terminate this\u00a0 contract without cause (solely by giving a notice and allowing a waiting\u00a0 period to expire)?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Rofr/Rofo/Rofn", "Description": "Is there a clause granting one party a right of first refusal, right of first offer or right of first negotiation to purchase, license, market, or distribute equity interest, technology, assets, products or services?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Change of Control", "Description": "Does one party have the right to terminate or is consent or notice required of the counterparty if such party undergoes a change of control, such as a merger, stock sale, transfer of all or substantially all of its assets or business, or assignment by operation of law?", "Answer Format": "Yes/No", "Group": "Group: 3", }, { "Category": "Anti-Assignment", "Description": "Is consent or notice required of a party if the contract is assigned to a third party?", "Answer Format": "Yes/No", "Group": "Group: 3", }, { "Category": "Revenue/Profit Sharing", "Description": "Is one party required to share revenue or profit with the counterparty for any technology, goods, or\u00a0services?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Price Restrictions", "Description": "Is there a restriction on the\u00a0 ability of a party to raise or reduce prices of technology, goods, or\u00a0 services provided?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Minimum Commitment", "Description": "Is there a minimum order size or minimum amount or units per-time period that one party must buy from the counterparty under the contract?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Volume Restriction", "Description": "Is there a fee increase or consent requirement, etc. if one party\u2019s use of the product/services exceeds certain threshold?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "IP Ownership Assignment", "Description": "Does intellectual property created\u00a0 by one party become the property of the counterparty, either per the terms of the contract or upon the occurrence of certain events?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Joint IP Ownership", "Description": "Is there any clause providing for joint or shared ownership of intellectual property between the parties to the contract?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "License Grant", "Description": "Does the contract contain a license granted by one party to its counterparty?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Non-Transferable License", "Description": "Does the contract limit the ability of a party to transfer the license being granted to a third party?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Affiliate License-Licensor", "Description": "Does the contract contain a license grant by affiliates of the licensor or that includes intellectual property of affiliates of the licensor?\u00a0", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Affiliate License-Licensee", "Description": "Does the contract contain a license grant to a licensee (incl. sublicensor) and the affiliates of such licensee/sublicensor?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Unlimited/All-You-Can-Eat-License", "Description": "Is there a clause granting one party an \u201centerprise,\u201d \u201call you can eat\u201d or unlimited usage license?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Irrevocable or Perpetual License", "Description": "Does the contract contain a\u00a0 license grant that is irrevocable or perpetual?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Source Code Escrow", "Description": "Is one party required to deposit its source code into escrow with a third party, which can be released to the counterparty upon the occurrence of certain events (bankruptcy,\u00a0 insolvency, etc.)?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Post-Termination Services", "Description": "Is a party subject to obligations after the termination or expiration of a contract, including any post-termination transition, payment, transfer of IP, wind-down, last-buy, or similar commitments?", "Answer Format": "Yes/No", "Group": "Group: 5", }, { "Category": "Audit Rights", "Description": "Does a party have the right to\u00a0 audit the books, records, or physical locations of the counterparty to ensure compliance with the contract?", "Answer Format": "Yes/No", "Group": "Group: 5", }, { "Category": "Uncapped Liability", "Description": "Is a party\u2019s liability uncapped upon the breach of its obligation in the contract? This also includes uncap liability for a particular type of breach such as IP infringement or breach of confidentiality obligation.", "Answer Format": "Yes/No", "Group": "Group: 6", }, { "Category": "Cap on Liability", "Description": "Does the contract include a cap on liability upon the breach of a party\u2019s obligation? This includes time limitation for the counterparty to bring claims or maximum amount for recovery.", "Answer Format": "Yes/No", "Group": "Group: 6", }, { "Category": "Liquidated Damages", "Description": "Does the contract contain a clause that would award either party liquidated damages for breach or a fee upon the termination of a contract (termination fee)?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Warranty Duration", "Description": "What is the duration of any\u00a0 warranty against defects or errors in technology, products, or services\u00a0 provided under the contract?", "Answer Format": "Number of months or years", "Group": "Group: -", }, { "Category": "Insurance", "Description": "Is there a requirement for insurance that must be maintained by one party for the benefit of the counterparty?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Covenant Not to Sue", "Description": "Is a party restricted from contesting the validity of the counterparty\u2019s ownership of intellectual property or otherwise bringing a claim against the counterparty for matters unrelated to the contract?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Third Party Beneficiary", "Description": "Is there a non-contracting party who is a beneficiary to some or all of the clauses in the contract and therefore can enforce its rights against a contracting party?", "Answer Format": "Yes/No", "Group": "Group: -", }, ] NUM_FIELDS_TO_EXTRACT_PER_CONTRACT = 41 # 0.15 is used in the Doc-ETL paper. It should be 0.5 for the actual benchmark. IOU_THRESH = 0.15 def get_label_df(num_contracts: int = 1, seed: int=42) -> pd.DataFrame: dataset = load_cuad_data(split="test") # get the set of unique contract titles; to ensure the order of the contracts is # preserved, we use a list rather than using python's set() contract_titles = [] for row in dataset: if row["title"] not in contract_titles: contract_titles.append(row["title"]) # shuffle the contracts for the given seed rng = np.random.default_rng(seed=seed) rng.shuffle(contract_titles) # get the first num_contracts contract_titles = contract_titles[:num_contracts] # construct the dataset one contract at a time final_label_dataset = [] for title in contract_titles: # get the rows for this contract contract_rows = [row for row in dataset if row["title"] == title] # construct the contract; we get the contract_id and contract text from the first row contract = { "contract_id": contract_rows[0]["id"], "title": title, "contract": contract_rows[0]["context"], } # add the labels category_names = list(map(lambda category: category["Category"], CUAD_CATEGORIES)) contract.update({category_name: [] for category_name in category_names}) for row in contract_rows: category_name = row["id"].split("__")[-1].split("_")[0].strip() category_name = category_name.replace(" For ", " for ") category_name = category_name.replace(" Of ", " of ") category_name = category_name.replace(" On ", " on ") category_name = category_name.replace(" Or ", " or ") category_name = category_name.replace(" To ", " to ") category_name = category_name.replace("Ip", "IP") assert category_name in category_names, f"Unknown category {category_name}" # Extract text from answers list (handles both old and new format) answer_texts = [] if isinstance(row["answers"], list): answer_texts = [ans["text"] for ans in row["answers"]] if row["answers"] else [] else: answer_texts = row["answers"].get("text", []) contract[category_name].extend(answer_texts) # add the contract to the dataset final_label_dataset.append(contract) return pd.DataFrame(final_label_dataset) # Return the Jaccard similarity between two strings def get_jaccard(label, pred): remove_tokens = [c for c in string.punctuation if c != "/"] for token in remove_tokens: label = label.replace(token, "") pred = pred.replace(token, "") label = label.lower() pred = pred.lower() label = label.replace("/", " ") pred = pred.replace("/", " ") label_words = set(label.split(" ")) pred_words = set(pred.split(" ")) intersection = label_words.intersection(pred_words) union = label_words.union(pred_words) jaccard = len(intersection) / len(union) return jaccard # Find the number of true positives, false positives, and false negatives for each entry # (one field extracted from each contract) by comparing the labels and predictions. # Labels and preds are lists of strings def evaluate_entry(labels, preds, substr_ok): tp, fp, fn = 0, 0, 0 # jaccard similarity expects strings # TODO: This is a hack, ideally, the return type of the preds should be known for idx, pred in enumerate(preds): if not isinstance(pred, str): print(f"Expected string, but got {pred}") preds[idx] = str(pred) # first check if labels is empty if len(labels) == 0: if len(preds) > 0: fp += len(preds) # false positive for each one else: for ans in labels: assert len(ans) > 0 # check if there is a match match_found = False for pred in preds: if substr_ok: is_match = get_jaccard(ans, pred) >= IOU_THRESH or ans in pred else: is_match = get_jaccard(ans, pred) >= IOU_THRESH if is_match: match_found = True if match_found: tp += 1 else: fn += 1 # now also get any fps by looping through preds for pred in preds: # Check if there's a match. if so, don't count (don't want to double count based on the above) # but if there's no match, then this is a false positive. # (Note: we get the true positives in the above loop instead of this loop so that we don't double count # multiple predictions that are matched with the same answer.) match_found = False for ans in labels: assert len(ans) > 0 if substr_ok: is_match = get_jaccard(ans, pred) >= IOU_THRESH or ans in pred else: is_match = get_jaccard(ans, pred) >= IOU_THRESH if is_match: match_found = True if not match_found: fp += 1 return tp, fp, fn def handle_empty_preds(preds): if preds is None or ( # noqa: SIM114 isinstance(preds, str) and (preds == "" or preds == " " or preds == "null" or preds == "None") ): return [] elif isinstance(preds, float) and np.isnan(preds): return [] if not isinstance(preds, (list, np.ndarray)): return [preds] return preds class CUADValidator(pz.Validator): def __init__(self, num_contracts: int = 1, seed: int=42): super().__init__() self.num_contracts = num_contracts self.seed = seed # get clean names for the categories self.category_names = list(map(lambda category: category["Category"], CUAD_CATEGORIES)) # compute mapping from contract_id --> label self.contract_id_to_label = self._compute_contract_id_to_labels() def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: tps, fps, fns = 0, 0, 0 for field in fields: preds = handle_empty_preds(output.get(field)) labels = self.contract_id_to_label[input_record["contract_id"]][field] entry_tp, entry_fp, entry_fn = evaluate_entry(labels, preds, substr_ok=True) if field == "Parties" else evaluate_entry(labels, preds, substr_ok=False) tps += entry_tp fps += entry_fp fns += entry_fn precision = tps / (tps + fps) if tps + fps > 0 else 0.0 recall = tps / (tps + fns) if tps + fns > 0 else 0.0 f1 = 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0.0 return f1 def _compute_contract_id_to_labels(self): # load full train dataset dataset = load_cuad_data(split="train") # get the set of unique contract titles; to ensure the order of the contracts is # preserved, we use a list rather than using python's set() contract_titles = [] for row in dataset: if row["title"] not in contract_titles: contract_titles.append(row["title"]) # shuffle the contracts for the given seed rng = np.random.default_rng(seed=self.seed) rng.shuffle(contract_titles) # get the first num_contracts contract_titles = contract_titles[:self.num_contracts] # construct the mapping from contract_id to labels contract_id_to_labels = {} for title in contract_titles: # get the rows for this contract contract_rows = [row for row in dataset if row["title"] == title] # get the contract_id from the first row contract_id = contract_rows[0]["id"] # get the labels labels = {category: [] for category in self.category_names} for row in contract_rows: category_name = row["id"].split("__")[-1].split("_")[0].strip() category_name = category_name.replace(" For ", " for ") category_name = category_name.replace(" Of ", " of ") category_name = category_name.replace(" On ", " on ") category_name = category_name.replace(" Or ", " or ") category_name = category_name.replace(" To ", " to ") category_name = category_name.replace("Ip", "IP") assert category_name in self.category_names, f"Unknown category {category_name}" # Extract text from answers list (handles both old and new format) answer_texts = [] if isinstance(row["answers"], list): answer_texts = [ans["text"] for ans in row["answers"]] if row["answers"] else [] else: answer_texts = row["answers"].get("text", []) labels[category_name].extend(answer_texts) # update the dictionary contract_id_to_labels[contract_id] = labels return contract_id_to_labels class CUADDataset(pz.IterDataset): def __init__(self, num_contracts: int = 1, split: str = "train", seed: int=42): self.num_contracts = num_contracts self.split = split self.seed = seed input_cols = [ {"name": "contract_id", "type": str, "desc": "The id of the the contract to be analyzed"}, {"name": "title", "type": str, "desc": "The title of the the contract to be analyzed"}, {"name": "contract", "type": str, "desc": "The content of the the contract to be analyzed"}, ] super().__init__(id="cuad", schema=input_cols) # convert the dataset into a list of dictionaries where each row is for a single contract dataset = load_cuad_data(split=split) self.dataset = self._construct_dataset(dataset, num_contracts, seed) def _construct_dataset(self, dataset, num_contracts, seed: int=42): # get the set of unique contract titles; to ensure the order of the contracts is # preserved, we use a list rather than using python's set() contract_titles = [] for row in dataset: if row["title"] not in contract_titles: contract_titles.append(row["title"]) # shuffle the contracts for the given seed rng = np.random.default_rng(seed=seed) rng.shuffle(contract_titles) # get the first num_contracts contract_titles = contract_titles[:num_contracts] # construct the dataset one contract at a time new_dataset = [] for title in contract_titles: # get the rows for this contract contract_rows = [row for row in dataset if row["title"] == title] # construct the contract; we get the contract_id and contract text from the first row contract = { "contract_id": contract_rows[0]["id"], "title": title, "contract": contract_rows[0]["context"], } # add the rows to the dataset new_dataset.append(contract) return new_dataset def __len__(self): return self.num_contracts def __getitem__(self, idx: int): return self.dataset[idx] # Compute the precision and recall for the entire dataset. # Each row in the dataframes should correspond to a contract. # The columns should be the extracted fields (categories in CUAD_CATEGORIES). def compute_precision_recall(label_df, preds_df): tp, fp, fn = 0, 0, 0 label_df = label_df.sort_values("contract_id").reset_index(drop=True) preds_df = preds_df.sort_values("contract_id").reset_index(drop=True) assert label_df.shape == preds_df.shape, ( f"Label and prediction dataframes have different shapes, label shape: {label_df.shape} vs preds shape {preds_df.shape}" ) categories = [category["Category"] for category in CUAD_CATEGORIES] for label_row, pred_row in zip(label_df.iterrows(), preds_df.iterrows()): assert label_row[1]["contract_id"] == pred_row[1]["contract_id"], ( f"IDs do not match. label id: {label_row[1]['contract_id']} vs pred id: {pred_row[1]['contract_id']}" ) for category in categories: substr_ok = "Parties" in category labels = label_row[1][category] assert isinstance(labels, list) preds = pred_row[1][category] preds = handle_empty_preds(preds) entry_tp, entry_fp, entry_fn = evaluate_entry(labels, preds, substr_ok) tp += entry_tp fp += entry_fp fn += entry_fn precision = tp / (tp + fp) if tp + fp > 0 else np.nan recall = tp / (tp + fn) if tp + fn > 0 else np.nan return precision, recall def parse_arguments(): parser = argparse.ArgumentParser(description="Run CUAD demo") parser.add_argument("--mode", type=str, help="one-convert or separate-converts", default="one-convert") parser.add_argument("--test", type=str, help="test time compute active or inactive", default="active") parser.add_argument("--constrained", default=False, action="store_true", help="Use constrained objective") parser.add_argument("--gpt4-mini-only", default=False, action="store_true", help="Use only GPT-4o-mini") parser.add_argument( "--sentinel-execution-strategy", default="mab", type=str, help="The engine to use. One of mab or random", ) parser.add_argument( "--execution-strategy", default="parallel", type=str, help="The plan executor to use. One of sequential, pipelined, parallel", ) parser.add_argument( "--optimizer-strategy", default="pareto", type=str, help="The optimizer to use. One of pareto or greedy", ) parser.add_argument("--verbose", default=False, action="store_true", help="Print verbose output") parser.add_argument( "--seed", default=42, type=int, help="Seed used to initialize RNG for MAB sampling algorithm", ) parser.add_argument( "--k", default=10, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--j", default=3, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--sample-budget", default=100, type=int, help="Total sample budget in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--exp-name", default=None, type=str, help="The experiment name.", ) parser.add_argument( "--priors-file", default=None, type=str, help="A file with a dictionary mapping physical operator ids to prior belief on their performance", ) parser.add_argument( "--quality", default=None, type=float, help="Quality threshold", ) parser.add_argument( "--policy", default="maxquality", type=str, help="One of 'mincost', 'mintime', 'maxquality'", ) return parser.parse_args() def build_cuad_query(dataset, mode): assert mode in ["one-convert", "separate-converts"] if mode == "one-convert": cols = [] for category in CUAD_CATEGORIES: desc = ( f"Extract the text spans (if they exist) from the contract corresponding to: {category['Description']}. If no spans exist, return an empty list. Quote text spans verbatim (do not summarize or paraphrase)." ) cols.append({"name": category["Category"], "type": list[str], "desc": desc}) desc = "Extract the text spans (if they exist) from the contract." dataset = dataset.sem_map(cols, depends_on=["contract"]) elif mode == "separate-converts": for category in CUAD_CATEGORIES: desc = ( f"Extract the text spans (if they exist) from the contract corresponding to: {category['Description']}. If no spans exist, return an empty list. Quote text spans verbatim (do not summarize or paraphrase)." ) dataset = dataset.sem_map( [{"name": category["Category"], "type": list[str], "desc": desc}], depends_on=["contract"], ) return dataset def main(): if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") args = parse_arguments() # create directory for profiling data os.makedirs("opt-profiling-data", exist_ok=True) # create validator for CUAD validator = CUADValidator(num_contracts=25, seed=args.seed) # create datasets for CUAD dataset = CUADDataset(split="test", num_contracts=100, seed=args.seed) train_dataset = CUADDataset(split="train", num_contracts=25, seed=args.seed) train_dataset = {train_dataset.id: train_dataset} print("Created datasets") # build and run the CUAD query query = build_cuad_query(dataset, args.mode) print("Built query; Starting query execution") # set the optimization policy; constraint set to 25% percentile from unconstrained plans policy = pz.MaxQuality() if not args.constrained else pz.MaxQualityAtFixedCost(max_cost=2.759) if args.policy == "mincost": policy = pz.MinCost() elif args.policy == "minlatency": policy = pz.MinTime() elif args.quality is not None and args.policy == "mincostatfixedquality": policy = pz.MinCostAtFixedQuality(min_quality=args.quality) elif args.quality is not None and args.policy == "minlatencyatfixedquality": policy = pz.MinTimeAtFixedQuality(min_quality=args.quality) print(f"USING POLICY: {policy}") # set models models = [Model.GPT_4o_MINI] if args.gpt4_mini_only else [ Model.GPT_4o, Model.GPT_4o_MINI, Model.LLAMA3_1_8B, Model.LLAMA3_3_70B, # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments` Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B, ] sentinel_strategy = args.sentinel_execution_strategy optimizer_strategy = args.optimizer_strategy execution_strategy = args.execution_strategy seed = args.seed k = args.k j = args.j sample_budget = args.sample_budget exp_name = ( f"cuad-final-{sentinel_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}" if args.exp_name is None else args.exp_name ) priors = None if args.priors_file is not None: with open(args.priors_file) as f: priors = json.load(f) config = pz.QueryProcessorConfig( policy=policy, verbose=False, optimizer_strategy=optimizer_strategy, sentinel_execution_strategy=sentinel_strategy, execution_strategy=execution_strategy, max_workers=64, available_models=models, allow_bonded_query=True, allow_critic=True, allow_mixtures=True, allow_rag_reduction=True, progress=True, k=k, j=j, sample_budget=sample_budget, seed=seed, exp_name=exp_name, priors=priors, dont_use_priors=(priors is None), ) print(f"EXPERIMENT NAME: {exp_name}") data_record_collection = query.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator) print("Query execution completed") # save statistics execution_stats_dict = data_record_collection.execution_stats.to_json() with open(f"opt-profiling-data/{exp_name}-stats.json", "w") as f: json.dump(execution_stats_dict, f) pred_df = data_record_collection.to_df() label_df = get_label_df(num_contracts=100, seed=seed) # pred_df.to_csv(f"{exp_name}-pred.csv", index=False) # label_df.to_csv(f"{exp_name}-label.csv", index=False) final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0] final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id] prec, recall = compute_precision_recall(label_df, pred_df) f1 = 2 * (prec * recall) / (prec + recall) if prec + recall > 0 else 0.0 stats_dict = { "precision": prec, "recall": recall, "f1": f1, "optimization_time": data_record_collection.execution_stats.optimization_time, "optimization_cost": data_record_collection.execution_stats.optimization_cost, "plan_execution_time": data_record_collection.execution_stats.plan_execution_time, "plan_execution_cost": data_record_collection.execution_stats.plan_execution_cost, "total_execution_time": data_record_collection.execution_stats.total_execution_time, "total_execution_cost": data_record_collection.execution_stats.total_execution_cost, "plan_str": final_plan_str, } with open(f"opt-profiling-data/{exp_name}-metrics.json", "w") as f: json.dump(stats_dict, f) print(f"Precision: {prec:.3f}, Recall: {recall:.3f}, F1: {f1:.3f}") print(f"Optimization time: {data_record_collection.execution_stats.optimization_time}") print(f"Optimization cost: {data_record_collection.execution_stats.optimization_cost}") print(f"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}") print(f"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}") print(f"Total Execution time: {data_record_collection.execution_stats.total_execution_time}") print(f"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}") if __name__ == "__main__": main() ================================================ FILE: abacus-research/cuad-max-quality-at-cost.py ================================================ import argparse import json import os import string import numpy as np import pandas as pd from cuad_data_loader import load_cuad_data import palimpzest as pz from palimpzest.constants import Model from palimpzest.policy import MaxQuality, MaxQualityAtFixedCost CUAD_CATEGORIES = [ { "Category": "Document Name", "Description": "The name of the contract", "Answer Format": "Contract Name", "Group": "Group: -", }, { "Category": "Parties", "Description": "The two or more parties who signed the contract", "Answer Format": "Entity or individual names", "Group": "Group: -", }, { "Category": "Agreement Date", "Description": "The date of the contract", "Answer Format": "Date (mm/dd/yyyy)", "Group": "Group: 1", }, { "Category": "Effective Date", "Description": "The date when the contract is effective\u00a0", "Answer Format": "Date (mm/dd/yyyy)", "Group": "Group: 1", }, { "Category": "Expiration Date", "Description": "On what date will the contract's initial term expire?", "Answer Format": "Date (mm/dd/yyyy) / Perpetual", "Group": "Group: 1", }, { "Category": "Renewal Term", "Description": "What is the renewal term after the initial term expires? This includes automatic extensions and unilateral extensions with prior notice.", "Answer Format": "[Successive] number of years/months / Perpetual", "Group": "Group: 1", }, { "Category": "Notice Period to Terminate Renewal", "Description": "What is the notice period required to terminate renewal?", "Answer Format": "Number of days/months/year(s)", "Group": "Group: 1", }, { "Category": "Governing Law", "Description": "Which state/country's law governs the interpretation of the contract?", "Answer Format": "Name of a US State / non-US Province, Country", "Group": "Group: -", }, { "Category": "Most Favored Nation", "Description": "Is there a clause that if a third party gets better terms on the licensing or sale of technology/goods/services described in the contract, the buyer of such technology/goods/services under the contract shall be entitled to those better terms?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Non-Compete", "Description": "Is there a restriction on the ability of a party to compete with the counterparty or operate in a certain geography or business or technology sector?\u00a0", "Answer Format": "Yes/No", "Group": "Group: 2", }, { "Category": "Exclusivity", "Description": "Is there an exclusive dealing\u00a0 commitment with the counterparty? This includes a commitment to procure all \u201crequirements\u201d from one party of certain technology, goods, or services or a prohibition on licensing or selling technology, goods or services to third parties, or a prohibition on\u00a0 collaborating or working with other parties), whether during the contract or\u00a0 after the contract ends (or both).", "Answer Format": "Yes/No", "Group": "Group: 2", }, { "Category": "No-Solicit of Customers", "Description": "Is a party restricted from contracting or soliciting customers or partners of the counterparty, whether during the contract or after the contract ends (or both)?", "Answer Format": "Yes/No", "Group": "Group: 2", }, { "Category": "Competitive Restriction Exception", "Description": "This category includes the exceptions or carveouts to Non-Compete, Exclusivity and No-Solicit of Customers above.", "Answer Format": "Yes/No", "Group": "Group: 2", }, { "Category": "No-Solicit of Employees", "Description": "Is there a restriction on a party\u2019s soliciting or hiring employees and/or contractors from the\u00a0 counterparty, whether during the contract or after the contract ends (or both)?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Non-Disparagement", "Description": "Is there a requirement on a party not to disparage the counterparty?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Termination for Convenience", "Description": "Can a party terminate this\u00a0 contract without cause (solely by giving a notice and allowing a waiting\u00a0 period to expire)?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Rofr/Rofo/Rofn", "Description": "Is there a clause granting one party a right of first refusal, right of first offer or right of first negotiation to purchase, license, market, or distribute equity interest, technology, assets, products or services?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Change of Control", "Description": "Does one party have the right to terminate or is consent or notice required of the counterparty if such party undergoes a change of control, such as a merger, stock sale, transfer of all or substantially all of its assets or business, or assignment by operation of law?", "Answer Format": "Yes/No", "Group": "Group: 3", }, { "Category": "Anti-Assignment", "Description": "Is consent or notice required of a party if the contract is assigned to a third party?", "Answer Format": "Yes/No", "Group": "Group: 3", }, { "Category": "Revenue/Profit Sharing", "Description": "Is one party required to share revenue or profit with the counterparty for any technology, goods, or\u00a0services?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Price Restrictions", "Description": "Is there a restriction on the\u00a0 ability of a party to raise or reduce prices of technology, goods, or\u00a0 services provided?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Minimum Commitment", "Description": "Is there a minimum order size or minimum amount or units per-time period that one party must buy from the counterparty under the contract?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Volume Restriction", "Description": "Is there a fee increase or consent requirement, etc. if one party\u2019s use of the product/services exceeds certain threshold?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "IP Ownership Assignment", "Description": "Does intellectual property created\u00a0 by one party become the property of the counterparty, either per the terms of the contract or upon the occurrence of certain events?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Joint IP Ownership", "Description": "Is there any clause providing for joint or shared ownership of intellectual property between the parties to the contract?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "License Grant", "Description": "Does the contract contain a license granted by one party to its counterparty?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Non-Transferable License", "Description": "Does the contract limit the ability of a party to transfer the license being granted to a third party?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Affiliate License-Licensor", "Description": "Does the contract contain a license grant by affiliates of the licensor or that includes intellectual property of affiliates of the licensor?\u00a0", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Affiliate License-Licensee", "Description": "Does the contract contain a license grant to a licensee (incl. sublicensor) and the affiliates of such licensee/sublicensor?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Unlimited/All-You-Can-Eat-License", "Description": "Is there a clause granting one party an \u201centerprise,\u201d \u201call you can eat\u201d or unlimited usage license?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Irrevocable or Perpetual License", "Description": "Does the contract contain a\u00a0 license grant that is irrevocable or perpetual?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Source Code Escrow", "Description": "Is one party required to deposit its source code into escrow with a third party, which can be released to the counterparty upon the occurrence of certain events (bankruptcy,\u00a0 insolvency, etc.)?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Post-Termination Services", "Description": "Is a party subject to obligations after the termination or expiration of a contract, including any post-termination transition, payment, transfer of IP, wind-down, last-buy, or similar commitments?", "Answer Format": "Yes/No", "Group": "Group: 5", }, { "Category": "Audit Rights", "Description": "Does a party have the right to\u00a0 audit the books, records, or physical locations of the counterparty to ensure compliance with the contract?", "Answer Format": "Yes/No", "Group": "Group: 5", }, { "Category": "Uncapped Liability", "Description": "Is a party\u2019s liability uncapped upon the breach of its obligation in the contract? This also includes uncap liability for a particular type of breach such as IP infringement or breach of confidentiality obligation.", "Answer Format": "Yes/No", "Group": "Group: 6", }, { "Category": "Cap on Liability", "Description": "Does the contract include a cap on liability upon the breach of a party\u2019s obligation? This includes time limitation for the counterparty to bring claims or maximum amount for recovery.", "Answer Format": "Yes/No", "Group": "Group: 6", }, { "Category": "Liquidated Damages", "Description": "Does the contract contain a clause that would award either party liquidated damages for breach or a fee upon the termination of a contract (termination fee)?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Warranty Duration", "Description": "What is the duration of any\u00a0 warranty against defects or errors in technology, products, or services\u00a0 provided under the contract?", "Answer Format": "Number of months or years", "Group": "Group: -", }, { "Category": "Insurance", "Description": "Is there a requirement for insurance that must be maintained by one party for the benefit of the counterparty?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Covenant Not to Sue", "Description": "Is a party restricted from contesting the validity of the counterparty\u2019s ownership of intellectual property or otherwise bringing a claim against the counterparty for matters unrelated to the contract?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Third Party Beneficiary", "Description": "Is there a non-contracting party who is a beneficiary to some or all of the clauses in the contract and therefore can enforce its rights against a contracting party?", "Answer Format": "Yes/No", "Group": "Group: -", }, ] NUM_FIELDS_TO_EXTRACT_PER_CONTRACT = 41 # 0.15 is used in the Doc-ETL paper. It should be 0.5 for the actual benchmark. IOU_THRESH = 0.15 def get_label_df(num_contracts: int = 1, seed: int=42) -> pd.DataFrame: dataset = load_cuad_data(split="test") # get the set of unique contract titles; to ensure the order of the contracts is # preserved, we use a list rather than using python's set() contract_titles = [] for row in dataset: if row["title"] not in contract_titles: contract_titles.append(row["title"]) # shuffle the contracts for the given seed rng = np.random.default_rng(seed=seed) rng.shuffle(contract_titles) # get the first num_contracts contract_titles = contract_titles[:num_contracts] # construct the dataset one contract at a time final_label_dataset = [] for title in contract_titles: # get the rows for this contract contract_rows = [row for row in dataset if row["title"] == title] # construct the contract; we get the contract_id and contract text from the first row contract = { "contract_id": contract_rows[0]["id"], "title": title, "contract": contract_rows[0]["context"], } # add the labels category_names = list(map(lambda category: category["Category"], CUAD_CATEGORIES)) for row in contract_rows: category_name = row["id"].split("__")[-1].split("_")[0].strip() category_name = category_name.replace(" For ", " for ") category_name = category_name.replace(" Of ", " of ") category_name = category_name.replace(" On ", " on ") category_name = category_name.replace(" Or ", " or ") category_name = category_name.replace(" To ", " to ") category_name = category_name.replace("Ip", "IP") assert category_name in category_names, f"Unknown category {category_name}" # Extract text from answers list (handles both old and new format) answer_texts = [] if isinstance(row["answers"], list): answer_texts = [ans["text"] for ans in row["answers"]] if row["answers"] else [] else: answer_texts = row["answers"].get("text", []) contract[category_name].extend(answer_texts) # add the contract to the dataset final_label_dataset.append(contract) return pd.DataFrame(final_label_dataset) # Return the Jaccard similarity between two strings def get_jaccard(label, pred): remove_tokens = [c for c in string.punctuation if c != "/"] for token in remove_tokens: label = label.replace(token, "") pred = pred.replace(token, "") label = label.lower() pred = pred.lower() label = label.replace("/", " ") pred = pred.replace("/", " ") label_words = set(label.split(" ")) pred_words = set(pred.split(" ")) intersection = label_words.intersection(pred_words) union = label_words.union(pred_words) jaccard = len(intersection) / len(union) return jaccard # Find the number of true positives, false positives, and false negatives for each entry # (one field extracted from each contract) by comparing the labels and predictions. # Labels and preds are lists of strings def evaluate_entry(labels, preds, substr_ok): tp, fp, fn = 0, 0, 0 # jaccard similarity expects strings # TODO: This is a hack, ideally, the return type of the preds should be known for idx, pred in enumerate(preds): if not isinstance(pred, str): print(f"Expected string, but got {pred}") preds[idx] = str(pred) # first check if labels is empty if len(labels) == 0: if len(preds) > 0: fp += len(preds) # false positive for each one else: for ans in labels: assert len(ans) > 0 # check if there is a match match_found = False for pred in preds: if substr_ok: is_match = get_jaccard(ans, pred) >= IOU_THRESH or ans in pred else: is_match = get_jaccard(ans, pred) >= IOU_THRESH if is_match: match_found = True if match_found: tp += 1 else: fn += 1 # now also get any fps by looping through preds for pred in preds: # Check if there's a match. if so, don't count (don't want to double count based on the above) # but if there's no match, then this is a false positive. # (Note: we get the true positives in the above loop instead of this loop so that we don't double count # multiple predictions that are matched with the same answer.) match_found = False for ans in labels: assert len(ans) > 0 if substr_ok: is_match = get_jaccard(ans, pred) >= IOU_THRESH or ans in pred else: is_match = get_jaccard(ans, pred) >= IOU_THRESH if is_match: match_found = True if not match_found: fp += 1 return tp, fp, fn def handle_empty_preds(preds): if preds is None or ( # noqa: SIM114 isinstance(preds, str) and (preds == "" or preds == " " or preds == "null" or preds == "None") ): return [] elif isinstance(preds, float) and np.isnan(preds): return [] if not isinstance(preds, (list, np.ndarray)): return [preds] return preds class CUADValidator(pz.Validator): def __init__(self, num_contracts: int = 1, seed: int=42): super().__init__() self.num_contracts = num_contracts self.seed = seed # get clean names for the categories self.category_names = list(map(lambda category: category["Category"], CUAD_CATEGORIES)) # compute mapping from contract_id --> label self.contract_id_to_label = self._compute_contract_id_to_labels() def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: scores = [] for field in fields: preds = handle_empty_preds(output.get(field)) labels = self.contract_id_to_label[input_record["contract_id"]][field] entry_tp, _, entry_fn = evaluate_entry(labels, preds, substr_ok=True) if field == "Parties" else evaluate_entry(labels, preds, substr_ok=False) score = None if len(labels) > 0: # noqa: SIM108 score = entry_tp / (entry_tp + entry_fn) else: score = 1.0 if len(preds) == 0 else 0.0 scores.append(score) return np.mean(scores) def _compute_contract_id_to_labels(self): # load full train dataset dataset = load_cuad_data(split="train") # get the set of unique contract titles; to ensure the order of the contracts is # preserved, we use a list rather than using python's set() contract_titles = [] for row in dataset: if row["title"] not in contract_titles: contract_titles.append(row["title"]) # shuffle the contracts for the given seed rng = np.random.default_rng(seed=self.seed) rng.shuffle(contract_titles) # get the first num_contracts contract_titles = contract_titles[:self.num_contracts] # construct the mapping from contract_id to labels contract_id_to_labels = {} for title in contract_titles: # get the rows for this contract contract_rows = [row for row in dataset if row["title"] == title] # get the contract_id from the first row contract_id = contract_rows[0]["id"] # get the labels labels = {category: [] for category in self.category_names} for row in contract_rows: category_name = row["id"].split("__")[-1].split("_")[0].strip() category_name = category_name.replace(" For ", " for ") category_name = category_name.replace(" Of ", " of ") category_name = category_name.replace(" On ", " on ") category_name = category_name.replace(" Or ", " or ") category_name = category_name.replace(" To ", " to ") category_name = category_name.replace("Ip", "IP") assert category_name in self.category_names, f"Unknown category {category_name}" # Extract text from answers list (handles both old and new format) answer_texts = [] if isinstance(row["answers"], list): answer_texts = [ans["text"] for ans in row["answers"]] if row["answers"] else [] else: answer_texts = row["answers"].get("text", []) labels[category_name].extend(answer_texts) # update the dictionary contract_id_to_labels[contract_id] = labels return contract_id_to_labels class CUADDataset(pz.IterDataset): def __init__(self, num_contracts: int = 1, split: str = "train", seed: int=42): self.num_contracts = num_contracts self.split = split self.seed = seed input_cols = [ {"name": "contract_id", "type": str, "desc": "The id of the the contract to be analyzed"}, {"name": "title", "type": str, "desc": "The title of the the contract to be analyzed"}, {"name": "contract", "type": str, "desc": "The content of the the contract to be analyzed"}, ] super().__init__(id="cuad", schema=input_cols) # Load dataset from local files dataset = load_cuad_data(split=split) self.dataset = self._construct_dataset(dataset, num_contracts, seed) def _construct_dataset(self, dataset, num_contracts, seed: int=42): # get the set of unique contract titles; to ensure the order of the contracts is # preserved, we use a list rather than using python's set() contract_titles = [] for row in dataset: if row["title"] not in contract_titles: contract_titles.append(row["title"]) # shuffle the contracts for the given seed rng = np.random.default_rng(seed=seed) rng.shuffle(contract_titles) # get the first num_contracts contract_titles = contract_titles[:num_contracts] # construct the dataset one contract at a time new_dataset = [] for title in contract_titles: # get the rows for this contract contract_rows = [row for row in dataset if row["title"] == title] # construct the contract; we get the contract_id and contract text from the first row contract = { "contract_id": contract_rows[0]["id"], "title": title, "contract": contract_rows[0]["context"], } # add the rows to the dataset new_dataset.append(contract) return new_dataset def __len__(self): return self.num_contracts def __getitem__(self, idx: int): return self.dataset[idx] # Compute the precision and recall for the entire dataset. # Each row in the dataframes should correspond to a contract. # The columns should be the extracted fields (categories in CUAD_CATEGORIES). def compute_precision_recall(label_df, preds_df): tp, fp, fn = 0, 0, 0 label_df = label_df.sort_values("contract_id").reset_index(drop=True) preds_df = preds_df.sort_values("contract_id").reset_index(drop=True) assert label_df.shape == preds_df.shape, ( f"Label and prediction dataframes have different shapes, label shape: {label_df.shape} vs preds shape {preds_df.shape}" ) categories = [category["Category"] for category in CUAD_CATEGORIES] for label_row, pred_row in zip(label_df.iterrows(), preds_df.iterrows()): assert label_row[1]["contract_id"] == pred_row[1]["contract_id"], ( f"IDs do not match. label id: {label_row[1]['contract_id']} vs pred id: {pred_row[1]['contract_id']}" ) for category in categories: substr_ok = "Parties" in category labels = label_row[1][category] assert isinstance(labels, list) preds = pred_row[1][category] preds = handle_empty_preds(preds) entry_tp, entry_fp, entry_fn = evaluate_entry(labels, preds, substr_ok) tp += entry_tp fp += entry_fp fn += entry_fn precision = tp / (tp + fp) if tp + fp > 0 else np.nan recall = tp / (tp + fn) if tp + fn > 0 else np.nan return precision, recall def parse_arguments(): parser = argparse.ArgumentParser(description="Run CUAD demo") parser.add_argument("--mode", type=str, help="one-convert or separate-converts", default="one-convert") parser.add_argument("--test", type=str, help="test time compute active or inactive", default="active") parser.add_argument("--constrained", default=False, action="store_true", help="Use constrained objective") parser.add_argument( "--sentinel-execution-strategy", default="mab", type=str, help="The engine to use. One of mab or random", ) parser.add_argument( "--optimizer-strategy", default="pareto", type=str, help="The optimizer to use. One of pareto or greedy", ) parser.add_argument("--verbose", default=False, action="store_true", help="Print verbose output") parser.add_argument( "--seed", default=42, type=int, help="Seed used to initialize RNG for MAB sampling algorithm", ) parser.add_argument( "--k", default=10, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--j", default=3, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--sample-budget", default=100, type=int, help="Total sample budget in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--exp-name", default=None, type=str, help="The experiment name.", ) parser.add_argument( "--priors-file", default=None, type=str, help="A file with a dictionary mapping physical operator ids to prior belief on their performance", ) parser.add_argument( "--cost", default=1.0, type=float, help="The cost budget for the optimization", ) return parser.parse_args() def build_cuad_query(dataset, mode): assert mode in ["one-convert", "separate-converts"] if mode == "one-convert": cols = [] for category in CUAD_CATEGORIES: desc = ( f"Extract the text spans (if they exist) from the contract corresponding to {category['Description']}" ) cols.append({"name": category["Category"], "type": list[str], "desc": desc}) desc = "Extract the text spans (if they exist) from the contract." dataset = dataset.sem_map(cols, desc=desc, depends_on=["contract"]) elif mode == "separate-converts": for category in CUAD_CATEGORIES: desc = ( f"Extract the text spans (if they exist) from the contract corresponding to {category['Description']}" ) dataset = dataset.sem_map( [{"name": category["Category"], "type": list[str], "desc": desc}], desc=category["Description"], depends_on=["contract"], ) return dataset def main(): if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") args = parse_arguments() # create directory for profiling data os.makedirs("max-quality-at-cost-data", exist_ok=True) # create validator for CUAD validator = CUADValidator(num_contracts=25, seed=args.seed) # create datasets for CUAD dataset = CUADDataset(split="test", num_contracts=100, seed=args.seed) train_dataset = CUADDataset(split="train", num_contracts=25, seed=args.seed) train_dataset = {train_dataset.id: train_dataset} print("Created datasets") # Build and run the CUAD query query = build_cuad_query(dataset, args.mode) print("Built query; Starting query execution") # set the optimization policy; constraint set to 25% percentile from unconstrained plans policy = MaxQualityAtFixedCost(max_cost=args.cost) if args.cost < 999 else MaxQuality() print(f"USING POLICY: {policy}") sentinel_strategy = args.sentinel_execution_strategy optimizer_strategy = args.optimizer_strategy seed = args.seed k = args.k j = args.j sample_budget = args.sample_budget exp_name = ( f"cuad-strategy-{optimizer_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}" if args.exp_name is None else args.exp_name ) priors = None if args.priors_file is not None: with open(args.priors_file) as f: priors = json.load(f) config = pz.QueryProcessorConfig( policy=policy, verbose=False, optimizer_strategy=optimizer_strategy, sentinel_execution_strategy=sentinel_strategy, execution_strategy="parallel", max_workers=64, available_models=[ Model.GPT_4o, Model.GPT_4o_MINI, Model.LLAMA3_1_8B, Model.LLAMA3_3_70B, # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments` Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B, ], allow_bonded_query=True, allow_critic=True, allow_mixtures=True, allow_rag_reduction=True, progress=True, seed=seed, k=k, j=j, sample_budget=sample_budget, exp_name=exp_name, priors=priors, ) print(f"EXPERIMENT NAME: {exp_name}") data_record_collection = query.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator) print("Query execution completed") # save statistics execution_stats_dict = data_record_collection.execution_stats.to_json() with open(f"max-quality-at-cost-data/{exp_name}-stats.json", "w") as f: json.dump(execution_stats_dict, f) pred_df = data_record_collection.to_df() label_df = get_label_df(num_contracts=100, seed=args.seed) # pred_df.to_csv(f"{exp_name}-pred.csv", index=False) # label_df.to_csv(f"{exp_name}-label.csv", index=False) final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0] final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id] prec, recall = compute_precision_recall(label_df, pred_df) f1 = 2 * (prec * recall) / (prec + recall) if prec + recall > 0 else 0.0 stats_dict = { "precision": prec, "recall": recall, "f1": f1, "optimization_time": data_record_collection.execution_stats.optimization_time, "optimization_cost": data_record_collection.execution_stats.optimization_cost, "plan_execution_time": data_record_collection.execution_stats.plan_execution_time, "plan_execution_cost": data_record_collection.execution_stats.plan_execution_cost, "total_execution_time": data_record_collection.execution_stats.total_execution_time, "total_execution_cost": data_record_collection.execution_stats.total_execution_cost, "plan_str": final_plan_str, } with open(f"max-quality-at-cost-data/{exp_name}-metrics.json", "w") as f: json.dump(stats_dict, f) print(f"Precision: {prec:.3f}, Recall: {recall:.3f}, F1: {f1:.3f}") print(f"Optimization time: {data_record_collection.execution_stats.optimization_time}") print(f"Optimization cost: {data_record_collection.execution_stats.optimization_cost}") print(f"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}") print(f"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}") print(f"Total Execution time: {data_record_collection.execution_stats.total_execution_time}") print(f"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}") if __name__ == "__main__": main() ================================================ FILE: abacus-research/cuad-priors.json ================================================ {"00c93aec22": {"quality": 0.5304878048780488, "cost": 0.01609626, "time": 121.91315126419067}, "00f4acd0d3": {"quality": 0.6195121951219512, "cost": 0.007308299999999999, "time": 62.89620461463928}, "0121878170": {"quality": 0.7871196283391406, "cost": 0.084214564, "time": 137.42955293655396}, "01c2f973ad": {"quality": 0.6369918699186992, "cost": 0.00766284, "time": 88.04302344322204}, "01fca3c717": {"quality": 0.6376887340301974, "cost": 0.067807144, "time": 56.765127992630006}, "02078988c1": {"quality": 0.6195121951219512, "cost": 0.010957832, "time": 58.52112374305725}, "021604dec1": {"quality": 0.32056910569105695, "cost": 0.014393556000000002, "time": 102.96775641441346}, "0262668df7": {"quality": 0.7521835075493613, "cost": 0.015505927999999999, "time": 110.59786329269409}, "0267c97b70": {"quality": 0.6195121951219512, "cost": 0.007206299999999999, "time": 78.4086401939392}, "02d6cdecdc": {"quality": 0.17902439024390246, "cost": 0.043331628, "time": 118.95345692634582}, "033ca325e6": {"quality": 0.6308943089430895, "cost": 0.006504208, "time": 47.695338296890256}, "0364b5e990": {"quality": 0.6195121951219512, "cost": 0.069325568, "time": 129.42635221481322}, "0375ea52c9": {"quality": 0.6893495934959348, "cost": 0.026699588000000003, "time": 43.96490340232849}, "038a5f0a62": {"quality": 0.6491056910569106, "cost": 0.01088205, "time": 142.37771649360656}, "039803b3b1": {"quality": 0.8145180023228804, "cost": 0.06754355, "time": 77.90279107093811}, "03b972cb56": {"quality": 0.20228803716608595, "cost": 0.04939144400000001, "time": 117.83176441192627}, "042d933706": {"quality": 0.6789430894308943, "cost": 0.011810856, "time": 96.84674339294433}, "050b21ce37": {"quality": 0.6331707317073171, "cost": 0.063467788, "time": 138.06499099731445}, "0524f42520": {"quality": 0.5098606271777004, "cost": 0.100158564, "time": 119.39450807571411}, "05420351e5": {"quality": 0.28818815331010456, "cost": 0.03700374, "time": 99.57810640335083}, "057e332ab1": {"quality": 0.5822299651567945, "cost": 0.071891988, "time": 137.52664232254028}, "0646f3f0fb": {"quality": 0.4296747967479675, "cost": 0.06800707800000001, "time": 122.81284818649291}, "06493715cc": {"quality": 0.6146341463414634, "cost": 0.009369455999999998, "time": 101.42031364440918}, "0659531b94": {"quality": 0.7977235772357723, "cost": 0.06570914000000001, "time": 88.527423286438}, "067ee6e91b": {"quality": 0.7303135888501743, "cost": 0.012997799999999999, "time": 91.4061038017273}, "0695f9b5fc": {"quality": 0.6195121951219512, "cost": 0.012743448, "time": 123.27798008918762}, "06e94a0f2e": {"quality": 0.6381300813008131, "cost": 0.022528364000000002, "time": 70.015203332901}, "073ef31d23": {"quality": 0.6711730545876887, "cost": 0.06321265200000001, "time": 125.18851342201233}, "078a7e545e": {"quality": 0.5136469221835075, "cost": 0.06231631000000001, "time": 139.78958024978638}, "079feb14a8": {"quality": 0.5636585365853659, "cost": 0.012793146, "time": 83.61773014068604}, "07a3a7daf7": {"quality": 0.6195121951219512, "cost": 0.00968809, "time": 64.83957290649414}, "08127cd6dd": {"quality": 0.6773170731707316, "cost": 0.018527102, "time": 74.52640342712402}, "0833133620": {"quality": 0.783089430894309, "cost": 0.011579322, "time": 103.93566818237305}, "087a2cabc4": {"quality": 0.7773054587688735, "cost": 0.041329712000000005, "time": 149.253262424469}, "08bf8cc191": {"quality": 0.6195121951219512, "cost": 0.018227304, "time": 146.171551656723}, "08e1802287": {"quality": 0.40414634146341466, "cost": 0.061857968, "time": 639.4221799850463}, "08f7f63b30": {"quality": 0.6039024390243902, "cost": 0.06393643200000002, "time": 150.91687684059144}, "090cd3ef31": {"quality": 0.5126016260162601, "cost": 0.024500464, "time": 242.8270983695984}, "0947216ece": {"quality": 0.6116260162601627, "cost": 0.022374364, "time": 86.5027738571167}, "096d51f670": {"quality": 0.6195121951219512, "cost": 0.015167303999999998, "time": 168.75218114852905}, "09791c731b": {"quality": 0.6195121951219512, "cost": 0.010591104, "time": 98.68934841156006}, "0990c0d4f8": {"quality": 0.7328455284552845, "cost": 0.015548904000000002, "time": 126.22301592826844}, "0a4c1bbb4a": {"quality": 0.6195121951219512, "cost": 0.01167957, "time": 112.43891968727112}, "0ac969dde3": {"quality": 0.532520325203252, "cost": 0.019075388000000002, "time": 182.81917290687562}, "0b4ab72197": {"quality": 0.6195121951219512, "cost": 0.01298358, "time": 88.19813923835754}, "0bf9d31691": {"quality": 0.6195121951219512, "cost": 0.06748344, "time": 112.7701108455658}, "0c020b86a3": {"quality": 0.48138211382113827, "cost": 0.012613356000000003, "time": 94.54495902061463}, "0c6c7fe96a": {"quality": 0.6439024390243903, "cost": 0.017969835999999996, "time": 175.61341876983641}, "0c81c8996a": {"quality": 0.6195121951219512, "cost": 0.008283972, "time": 123.77774615287781}, "0cd25da9fe": {"quality": 0.41504065040650406, "cost": 0.071906784, "time": 151.1712607383728}, "0cd78f33d8": {"quality": 0.6969337979094077, "cost": 0.12822476, "time": 110.73728203773499}, "0cdc5954dd": {"quality": 0.39105691056910574, "cost": 0.013535116, "time": 110.69660997390747}, "0d53dd53c1": {"quality": 0.4902439024390244, "cost": 0.08892936000000001, "time": 129.61446180343628}, "0d8436af32": {"quality": 0.6047386759581882, "cost": 0.03048576, "time": 119.35665726661682}, "0e38896654": {"quality": 0.7874680603948896, "cost": 0.029542839999999997, "time": 118.93085074424744}, "0ec672e7c8": {"quality": 0.3153774680603949, "cost": 0.018613362, "time": 156.979243183136}, "0ed243f788": {"quality": 0.6195121951219512, "cost": 0.014807712, "time": 171.58208327293397}, "0eeb372802": {"quality": 0.6195121951219512, "cost": 0.017926556, "time": 135.83013033866882}, "0ef0becc1b": {"quality": 0.3480371660859466, "cost": 0.058053114, "time": 112.96433029174804}, "0fefead197": {"quality": 0.7968176538908246, "cost": 0.09321153600000001, "time": 177.5354432106018}, "0ff126ebf8": {"quality": 0.3920325203252033, "cost": 0.01680763, "time": 116.37514338493347}, "10d1d4bdeb": {"quality": 0.6195121951219512, "cost": 0.066024524, "time": 129.0803412914276}, "114a097c53": {"quality": 0.6195121951219512, "cost": 0.00815448, "time": 68.65160498619079}, "1175ee37e6": {"quality": 0.5414634146341463, "cost": 0.002796924, "time": 66.07583198547363}, "11bc996d48": {"quality": 0.7575261324041811, "cost": 0.011037809999999999, "time": 160.15645961761476}, "11ded03305": {"quality": 0.7726016260162601, "cost": 0.07187866600000001, "time": 108.60114121437073}, "123fb650fb": {"quality": 0.6260162601626017, "cost": 0.012730068, "time": 158.32172050476075}, "132f6f3946": {"quality": 0.6478048780487804, "cost": 0.06792896600000001, "time": 142.56146936416627}, "133ee5023f": {"quality": 0.6323577235772359, "cost": 0.009341408, "time": 76.29307880401612}, "13a009fe0c": {"quality": 0.6134146341463416, "cost": 0.020489088, "time": 159.91688990592957}, "13da306f84": {"quality": 0.504390243902439, "cost": 0.013949744, "time": 125.56137013435364}, "13e717e41e": {"quality": 0.5087804878048781, "cost": 0.055634039999999996, "time": 89.58166871070861}, "13f75f9bd0": {"quality": 0.3702439024390244, "cost": 0.009800003999999998, "time": 140.3795708656311}, "140ededb41": {"quality": 0.7700813008130082, "cost": 0.0035910299999999994, "time": 54.85756406784058}, "142e59c03f": {"quality": 0.39747967479674795, "cost": 0.080664288, "time": 125.91337852478027}, "142f3a7c70": {"quality": 0.6195121951219512, "cost": 0.009178488000000002, "time": 142.1152557849884}, "1468dddecc": {"quality": 0.6699186991869919, "cost": 0.007540319999999999, "time": 112.98228130340576}, "15af009a01": {"quality": 0.47317073170731705, "cost": 0.07114547600000001, "time": 165.90874967575073}, "15b80a55d3": {"quality": 0.5414634146341464, "cost": 0.06812305999999999, "time": 156.45735163688659}, "1625e624c5": {"quality": 0.56931475029036, "cost": 0.05795948000000001, "time": 88.15128273963929}, "16cff1c1e9": {"quality": 0.6195121951219512, "cost": 0.06384869200000001, "time": 149.38432698249818}, "17407df027": {"quality": 0.5050406504065041, "cost": 0.06935714000000001, "time": 157.59272437095643}, "176da24f53": {"quality": 0.6195121951219512, "cost": 0.006307396000000002, "time": 48.773886156082156}, "179379555f": {"quality": 0.43966318234610924, "cost": 0.008765568, "time": 131.12133555412294}, "181c91d1be": {"quality": 0.4508943089430894, "cost": 0.016041972, "time": 104.77644038200378}, "183743e76e": {"quality": 0.6195121951219512, "cost": 0.007336812, "time": 88.6048484802246}, "186b58c209": {"quality": 0.47317073170731705, "cost": 0.064688372, "time": 136.70242128372192}, "187eace9fe": {"quality": 0.6195121951219512, "cost": 0.061498704, "time": 111.12849621772766}, "190ed2e1b6": {"quality": 0.4953193960511033, "cost": 0.06357867000000002, "time": 139.53685355186462}, "191aafe1a6": {"quality": 0.6285017421602788, "cost": 0.05959545200000001, "time": 133.747340965271}, "194919ad28": {"quality": 0.6195121951219512, "cost": 0.061897668, "time": 120.82026715278626}, "197bb53f10": {"quality": 0.4398373983739837, "cost": 0.012262326, "time": 128.21201944351196}, "19ba7d0617": {"quality": 0.8144715447154471, "cost": 0.18371900000000002, "time": 127.1558424949646}, "19e3db7fe7": {"quality": 0.6195121951219512, "cost": 0.016522608, "time": 157.01711511611938}, "1a08cb3f50": {"quality": 0.6195121951219512, "cost": 0.06344755, "time": 113.1626263141632}, "1a169179f6": {"quality": 0.3619976771196284, "cost": 0.096862584, "time": 173.04007925987244}, "1a71d61ac4": {"quality": 0.6055284552845528, "cost": 0.044680528000000004, "time": 134.11820168495177}, "1ad856985f": {"quality": 0.6195121951219512, "cost": 0.015116656000000001, "time": 213.2139684677124}, "1adec2dca2": {"quality": 0.6195121951219512, "cost": 0.018743704, "time": 189.0085569858551}, "1b04a2b184": {"quality": 0.27695702671312433, "cost": 0.018901547999999997, "time": 190.3927951812744}, "1b28439bd7": {"quality": 0.595040650406504, "cost": 0.018163196000000003, "time": 130.19455728530883}, "1b2c667b15": {"quality": 0.6146341463414634, "cost": 0.06703916, "time": 141.4092381477356}, "1b4511eada": {"quality": 0.5601277584204414, "cost": 0.06213515000000001, "time": 197.098247051239}, "1b7e6cad66": {"quality": 0.6678048780487805, "cost": 0.061792784, "time": 95.13608679771423}, "1beb2fac62": {"quality": 0.5772357723577235, "cost": 0.023681768000000002, "time": 99.76408967971801}, "1c347e4d91": {"quality": 0.5130894308943089, "cost": 0.005374038, "time": 119.8269275188446}, "1c35bf4be6": {"quality": 0.6613124274099883, "cost": 0.06615356000000001, "time": 178.00547165870665}, "1c4bbf8f7e": {"quality": 0.812137049941928, "cost": 0.09307272000000003, "time": 165.35304036140442}, "1c5f1341f6": {"quality": 0.6847967479674797, "cost": 0.035318894, "time": 106.12908926010132}, "1c71804bec": {"quality": 0.5271544715447154, "cost": 0.037727392, "time": 100.24818325042725}, "1cc6d9efb6": {"quality": 0.4643902439024391, "cost": 0.018275512, "time": 88.59051547050476}, "1ce3d77039": {"quality": 0.6304878048780488, "cost": 0.0014104099999999997, "time": 62.92718415260315}, "1d26090364": {"quality": 0.5578048780487805, "cost": 0.00619749, "time": 73.83342995643616}, "1d87f97e62": {"quality": 0.6894192799070848, "cost": 0.013183738, "time": 122.42380046844482}, "1d90fb8ca6": {"quality": 0.7902903600464576, "cost": 0.08424590000000001, "time": 134.87732191085814}, "1da2369719": {"quality": 0.676492450638792, "cost": 0.010263576, "time": 102.77953281402588}, "1e18e60895": {"quality": 0.6342276422764228, "cost": 0.0014448139999999998, "time": 62.875070858001706}, "1e1bf7e88b": {"quality": 0.38325203252032525, "cost": 0.01685045, "time": 99.01278185844421}, "1e8b3521f8": {"quality": 0.6149709639953542, "cost": 0.006865926, "time": 112.37140879631042}, "1f412964ff": {"quality": 0.6195121951219512, "cost": 0.059373496000000005, "time": 141.1672589302063}, "1f72cfb78a": {"quality": 0.2634494773519164, "cost": 0.04392928, "time": 141.55319528579713}, "1fb5d170ad": {"quality": 0.4796399535423926, "cost": 0.061984526000000005, "time": 130.7815794467926}, "20180dd292": {"quality": 0.6195121951219512, "cost": 0.06752007200000001, "time": 180.62230010032653}, "2018bef45f": {"quality": 0.624390243902439, "cost": 0.010508668, "time": 100.28150401115417}, "2075ff1d04": {"quality": 0.3623577235772358, "cost": 0.015088508000000004, "time": 1278.50945892334}, "208a98f514": {"quality": 0.775156794425087, "cost": 0.017165512, "time": 166.07545523643495}, "20904e5c14": {"quality": 0.476260162601626, "cost": 0.072447476, "time": 98.36213431358337}, "20afc3d539": {"quality": 0.5903716608594658, "cost": 0.084462272, "time": 125.81502628326416}, "20e10af7d4": {"quality": 0.6369918699186992, "cost": 0.011088839999999999, "time": 154.89131064414977}, "20e2c0b057": {"quality": 0.7137398373983739, "cost": 0.014822027999999998, "time": 160.84442710876465}, "211b89b4cd": {"quality": 0.3874680603948897, "cost": 0.0074940599999999994, "time": 110.43994216918945}, "21386082aa": {"quality": 0.44699186991869916, "cost": 0.04453856, "time": 132.42854318618774}, "21b2b8ebd1": {"quality": 0.4067479674796748, "cost": 0.010719648000000002, "time": 121.89121770858765}, "21b78249a7": {"quality": 0.40636469221835075, "cost": 0.08786833200000001, "time": 114.88702239990235}, "21bed16a7d": {"quality": 0.42804878048780487, "cost": 0.021284448, "time": 757.0631816387177}, "2200d969d0": {"quality": 0.729616724738676, "cost": 0.103071816, "time": 184.93275413513183}, "220d008704": {"quality": 0.6195121951219512, "cost": 0.061679156, "time": 194.95589275360106}, "2251d21392": {"quality": 0.3285365853658536, "cost": 0.063983622, "time": 125.72053866386413}, "23566f15ab": {"quality": 0.6269918699186992, "cost": 0.0011016140000000001, "time": 68.13801875114441}, "23a9506d36": {"quality": 0.6296747967479674, "cost": 0.04247636400000001, "time": 287.74875631332395}, "24957f3a43": {"quality": 0.36911730545876886, "cost": 0.007789104, "time": 66.6147982120514}, "2529e2f8b0": {"quality": 0.6348780487804878, "cost": 0.018974088000000004, "time": 46.99674005508423}, "252f01ac5b": {"quality": 0.7063414634146341, "cost": 0.09021934000000001, "time": 138.44833331108094}, "25fadf0883": {"quality": 0.6009756097560977, "cost": 0.069262768, "time": 195.04961967468262}, "2609bfd616": {"quality": 0.4904878048780488, "cost": 0.014383092, "time": 138.67839546203612}, "2629f3e324": {"quality": 0.3373867595818815, "cost": 0.06496914000000001, "time": 199.1607180118561}, "262e4298f9": {"quality": 0.5845528455284553, "cost": 0.06646106, "time": 168.6952573776245}, "26cc40d3bb": {"quality": 0.6195121951219512, "cost": 0.066186748, "time": 191.06331505775452}, "2728c8eb6a": {"quality": 0.6195121951219512, "cost": 0.013364808, "time": 136.0991479873657}, "27bc52befa": {"quality": 0.7133333333333334, "cost": 0.035852640000000005, "time": 102.54774498939514}, "27daa50458": {"quality": 0.5414634146341464, "cost": 0.008815032, "time": 95.5854898929596}, "2821795e69": {"quality": 0.47317073170731716, "cost": 0.06960079, "time": 199.99654774665834}, "28369b2421": {"quality": 0.5271544715447154, "cost": 0.012585374, "time": 139.39593086242675}, "28421e6d62": {"quality": 0.6195121951219512, "cost": 0.010284108, "time": 126.62169485092163}, "2936c3e43e": {"quality": 0.7311382113821139, "cost": 0.022435948, "time": 190.63684725761414}, "293ec5edca": {"quality": 0.8191753774680605, "cost": 0.014760236, "time": 130.9795027732849}, "29409d0894": {"quality": 0.30975609756097555, "cost": 0.023374712000000002, "time": 67.99601030349731}, "294258298a": {"quality": 0.6918466898954704, "cost": 0.060563396000000005, "time": 171.09867010116577}, "294e541235": {"quality": 0.6195121951219512, "cost": 0.004267548, "time": 63.70019145011902}, "295ed5e759": {"quality": 0.2658536585365854, "cost": 0.00976596, "time": 102.0877944469452}, "2960431101": {"quality": 0.40162601626016264, "cost": 0.010855686, "time": 106.73301725387573}, "29892d8468": {"quality": 0.6195121951219512, "cost": 0.016309508, "time": 152.2814826965332}, "299a0aeb65": {"quality": 0.6601509872241579, "cost": 0.014925237999999999, "time": 216.44449853897095}, "29ad99e3ed": {"quality": 0.6195121951219512, "cost": 0.06017612800000001, "time": 140.98456511497497}, "2a5edac2de": {"quality": 0.4360046457607433, "cost": 0.093795708, "time": 186.12790670394898}, "2a7d15f4a7": {"quality": 0.6637398373983739, "cost": 0.0017069580000000002, "time": 66.5458746433258}, "2aa996de6a": {"quality": 0.40952380952380957, "cost": 0.09028132800000001, "time": 171.12979340553284}, "2ac4fb293f": {"quality": 0.6234146341463415, "cost": 0.039522680000000004, "time": 134.96180925369262}, "2afeff0083": {"quality": 0.5430894308943089, "cost": 0.065989844, "time": 233.06356620788574}, "2b2bc9568b": {"quality": 0.6195121951219512, "cost": 0.071159096, "time": 242.5479420185089}, "2b5679d248": {"quality": 0.6195121951219512, "cost": 0.018697384, "time": 247.21016097068787}, "2bcf54cda1": {"quality": 0.5373983739837398, "cost": 0.06900906000000001, "time": 200.87765913009645}, "2bd39ee744": {"quality": 0.7727642276422764, "cost": 0.029763757999999994, "time": 142.61127648353576}, "2bf38d797f": {"quality": 0.6145528455284553, "cost": 0.015432928000000002, "time": 154.4491545200348}, "2c4f4f304e": {"quality": 0.5686875725900116, "cost": 0.02265774, "time": 104.49429354667663}, "2c5cf9eb26": {"quality": 0.6195121951219512, "cost": 0.010875623999999999, "time": 208.2448058128357}, "2c9a9f94c4": {"quality": 0.6195121951219512, "cost": 0.016278144, "time": 211.14152693748474}, "2d3bbc2d23": {"quality": 0.5347386759581882, "cost": 0.011645886, "time": 142.9862838745117}, "2de113167b": {"quality": 0.3548780487804878, "cost": 0.01615302, "time": 215.62943921089172}, "2de3eb2c19": {"quality": 0.6495934959349594, "cost": 0.012367168000000001, "time": 84.4677538394928}, "2e30394ac6": {"quality": 0.42853658536585365, "cost": 0.013055616, "time": 162.57028393745424}, "2e9c5cc9bf": {"quality": 0.6195121951219512, "cost": 0.008993688000000001, "time": 123.94604053497315}, "2f1573da80": {"quality": 0.628048780487805, "cost": 0.012240792, "time": 185.1746757030487}, "2f39d78f34": {"quality": 0.6814169570267131, "cost": 0.08897421200000001, "time": 171.90278725624086}, "2fc0cb3592": {"quality": 0.3300813008130082, "cost": 0.011238624, "time": 162.5598623752594}, "2fd9cd426a": {"quality": 0.3750406504065041, "cost": 0.01883974, "time": 179.53315086364745}, "300924ebae": {"quality": 0.2581533101045296, "cost": 0.08039779600000001, "time": 140.4485348701477}, "3019af79b3": {"quality": 0.5535423925667828, "cost": 0.014683848, "time": 1308.0872743606567}, "302c1d97fc": {"quality": 0.6195121951219512, "cost": 0.01627172, "time": 136.45579180717468}, "30ae4cbe91": {"quality": 0.37552845528455286, "cost": 0.006890768, "time": 750.692762708664}, "30c1f9ddf1": {"quality": 0.6262369337979093, "cost": 0.071551416, "time": 112.11581921577454}, "30cd375570": {"quality": 0.6195121951219512, "cost": 0.06602648800000002, "time": 118.63089265823365}, "3169782cbb": {"quality": 0.4402439024390244, "cost": 0.060568003999999995, "time": 103.64021511077881}, "3172fc459a": {"quality": 0.4475609756097561, "cost": 0.073885504, "time": 161.98054652214051}, "318499c14b": {"quality": 0.4873054587688733, "cost": 0.06648116000000001, "time": 187.55820865631102}, "31a32be94d": {"quality": 0.8294541231126598, "cost": 0.08899002, "time": 188.3829703807831}, "32b101d807": {"quality": 0.6408594657375145, "cost": 0.023361424, "time": 210.3520890712738}, "32e2c7ad7f": {"quality": 0.317479674796748, "cost": 0.06465559000000001, "time": 176.4482195854187}, "33459cd29c": {"quality": 0.6195121951219512, "cost": 0.014899176000000004, "time": 156.23946170806886}, "33a187e74f": {"quality": 0.6709756097560975, "cost": 0.01123047, "time": 201.9312825202942}, "33bab4f766": {"quality": 0.7291869918699188, "cost": 0.063704096, "time": 215.9541217803955}, "34922140da": {"quality": 0.5666666666666667, "cost": 0.06817234800000001, "time": 192.9766547679901}, "3511b5e1d0": {"quality": 0.6195121951219512, "cost": 0.013343184, "time": 129.7560082912445}, "3513311c2d": {"quality": 0.48170731707317077, "cost": 0.064084452, "time": 141.37073793411255}, "3513e54767": {"quality": 0.6195121951219512, "cost": 0.010438112, "time": 88.52075481414795}, "353f0cb1ac": {"quality": 0.6175609756097561, "cost": 0.006175324, "time": 85.6404734134674}, "3550bf88cb": {"quality": 0.6732752613240418, "cost": 0.012484512, "time": 185.91844930648804}, "35610fb420": {"quality": 0.7706271777003485, "cost": 0.009480828, "time": 151.24167275428772}, "357267e14b": {"quality": 0.40682926829268296, "cost": 0.017491424, "time": 768.3776504993439}, "35baa5c3cc": {"quality": 0.7191056910569106, "cost": 0.07052012, "time": 176.32511940002442}, "3637084f91": {"quality": 0.5414634146341464, "cost": 0.010371784, "time": 99.66692337989807}, "368a497102": {"quality": 0.5840650406504064, "cost": 0.039775324, "time": 107.50217499732972}, "36c66671ee": {"quality": 0.6195121951219512, "cost": 0.010492836, "time": 127.38456826210022}, "37456cb002": {"quality": 0.800801393728223, "cost": 0.036142828, "time": 157.18044147491455}, "3746ea5c03": {"quality": 0.45056910569105685, "cost": 0.03366273200000001, "time": 102.54732451438903}, "375ed248fe": {"quality": 0.33772357723577234, "cost": 0.013014735999999999, "time": 85.65126795768738}, "377cdf9209": {"quality": 0.7822648083623693, "cost": 0.036440860000000005, "time": 189.91213726997375}, "37d4d0f214": {"quality": 0.6195121951219512, "cost": 0.071572196, "time": 152.68034386634827}, "37ece7217f": {"quality": 0.47804878048780486, "cost": 0.05685350800000001, "time": 76.75442943572997}, "38075bb01f": {"quality": 0.6195121951219512, "cost": 0.010187640000000001, "time": 148.4516242980957}, "3831d758b1": {"quality": 0.6195121951219512, "cost": 0.063214564, "time": 129.3372209072113}, "38567d6a43": {"quality": 0.6195121951219512, "cost": 0.008923896, "time": 136.575146484375}, "3875787727": {"quality": 0.6195121951219512, "cost": 0.06695744000000001, "time": 189.5042601108551}, "389c54cbca": {"quality": 0.633739837398374, "cost": 0.010009344, "time": 94.1163019657135}, "3980f20caa": {"quality": 0.5065040650406505, "cost": 0.020949968, "time": 150.0350682735443}, "3997a836bd": {"quality": 0.7323577235772357, "cost": 0.069346092, "time": 181.2115756034851}, "39ad76f8ce": {"quality": 0.5396747967479676, "cost": 0.06732360800000001, "time": 117.92112016677856}, "39c0b7c171": {"quality": 0.4475609756097561, "cost": 0.07215601600000002, "time": 221.94310250282288}, "39cd4ca402": {"quality": 0.6195121951219512, "cost": 0.007245383999999999, "time": 139.3380250453949}, "3a32c98a53": {"quality": 0.7996399535423925, "cost": 0.066284192, "time": 151.66175842285156}, "3ac7fa4e46": {"quality": 0.6183972125435541, "cost": 0.09699730400000002, "time": 154.47008938789367}, "3ad6dcf559": {"quality": 0.6195121951219512, "cost": 0.071971364, "time": 124.51828866004944}, "3ae0de8663": {"quality": 0.2573170731707317, "cost": 0.043973896, "time": 145.02047486305236}, "3b2e8075ea": {"quality": 0.7909523809523809, "cost": 0.007301412, "time": 154.77439031600952}, "3b3676521a": {"quality": 0.7468989547038328, "cost": 0.025589519999999998, "time": 245.33448405265807}, "3b57530a56": {"quality": 0.7408943089430895, "cost": 0.009371704, "time": 124.3612250328064}, "3b6fbfa11d": {"quality": 0.4241695702671313, "cost": 0.03699336, "time": 136.01204528808594}, "3b81215e7a": {"quality": 0.7074099883855982, "cost": 0.069720068, "time": 194.1675964832306}, "3b9f8045d7": {"quality": 0.5806620209059233, "cost": 0.09104832400000001, "time": 150.99677429199218}, "3c5857683c": {"quality": 0.5024390243902439, "cost": 0.07306957600000001, "time": 161.98392362594603}, "3cbab8082e": {"quality": 0.6873751451800232, "cost": 0.014340899999999998, "time": 174.04833178520204}, "3d21104666": {"quality": 0.24378629500580723, "cost": 0.126616, "time": 80.24543118476868}, "3d71c4dd2c": {"quality": 0.6786062717770036, "cost": 0.010625964, "time": 140.9857957839966}, "3d9e24215e": {"quality": 0.3852845528455284, "cost": 0.028569556000000003, "time": 57.36162734031677}, "3e7efee65a": {"quality": 0.5565040650406504, "cost": 0.066932276, "time": 163.1447666168213}, "3ed0ad20ed": {"quality": 0.32991869918699185, "cost": 0.071716344, "time": 165.24500761032104}, "3f1a58aec9": {"quality": 0.533739837398374, "cost": 0.013489895999999998, "time": 331.1659511566162}, "3f2b07cb78": {"quality": 0.19126596980255517, "cost": 0.041601888, "time": 124.82682638168335}, "3f3ef494b0": {"quality": 0.6195121951219512, "cost": 0.0015848999999999998, "time": 51.92140054702759}, "3f62c3fbfc": {"quality": 0.19586527293844366, "cost": 0.0071477519999999985, "time": 118.16850218772888}, "3f730d8bfe": {"quality": 0.7800813008130081, "cost": 0.083953044, "time": 111.95527267456055}, "3f8d2ee81f": {"quality": 0.6478048780487805, "cost": 0.042814124, "time": 129.17971215248107}, "40104c813f": {"quality": 0.6195121951219512, "cost": 0.016629716, "time": 190.76988382339476}, "403b05da2d": {"quality": 0.6195121951219512, "cost": 0.013792372, "time": 143.9212529182434}, "403f0726fa": {"quality": 0.44464576074332174, "cost": 0.07387800000000001, "time": 62.18464155197144}, "4098178354": {"quality": 0.6802322880371661, "cost": 0.06734463600000001, "time": 214.58685207366943}, "409ff67607": {"quality": 0.6195121951219512, "cost": 0.0056238419999999996, "time": 99.12217197418212}, "40b3b6642c": {"quality": 0.4744831591173055, "cost": 0.06449838599999999, "time": 133.02915487289428}, "412c065b83": {"quality": 0.6195121951219512, "cost": 0.009992927999999998, "time": 171.42633333206177}, "4191118787": {"quality": 0.6195121951219512, "cost": 0.014096268, "time": 191.99022674560547}, "41d5b97871": {"quality": 0.6195121951219512, "cost": 0.003545856, "time": 92.14297785758973}, "41d8845655": {"quality": 0.7837282229965157, "cost": 0.021811158000000004, "time": 109.06256651878357}, "41ee202cac": {"quality": 0.5507317073170731, "cost": 0.014144588000000003, "time": 1669.106618309021}, "41fe4aee55": {"quality": 0.6195121951219512, "cost": 0.06379543600000001, "time": 147.32905130386354}, "42430ea391": {"quality": 0.39349593495934954, "cost": 0.073369736, "time": 190.46762056350707}, "42ddd48341": {"quality": 0.4267479674796748, "cost": 0.023891804, "time": 166.09094681739808}, "42f1e19aa7": {"quality": 0.2668060394889663, "cost": 0.10482261600000001, "time": 180.36561794281005}, "430a2ab32f": {"quality": 0.49148664343786297, "cost": 0.08972675600000002, "time": 123.45088205337524}, "4339427ad8": {"quality": 0.6195121951219512, "cost": 0.06641424400000001, "time": 176.55676898956298}, "4361bc7ea7": {"quality": 0.433472706155633, "cost": 0.018289859999999998, "time": 1338.038490152359}, "43c3cf9cb8": {"quality": 0.6104065040650406, "cost": 0.011720016, "time": 66.8409245967865}, "43d24fb32a": {"quality": 0.3804878048780488, "cost": 0.016379803999999998, "time": 121.97038044929505}, "43e9b39e5c": {"quality": 0.7054587688734031, "cost": 0.07738095200000002, "time": 196.7220965385437}, "44d6af5523": {"quality": 0.6940650406504065, "cost": 0.010872959999999997, "time": 159.60600094795228}, "44f189d813": {"quality": 0.7999303135888501, "cost": 0.038299172000000006, "time": 177.3291199207306}, "450f45a187": {"quality": 0.6195121951219512, "cost": 0.016291588000000003, "time": 72.91690697669983}, "453d0a5097": {"quality": 0.39195121951219514, "cost": 0.029831792000000003, "time": 120.01631045341492}, "4547ef4c8e": {"quality": 0.4353658536585366, "cost": 0.065468638, "time": 134.60169172286987}, "461846a52d": {"quality": 0.6195121951219512, "cost": 0.006258096, "time": 108.38983693122864}, "462e6ff849": {"quality": 0.4353658536585366, "cost": 0.016916924, "time": 133.2640913963318}, "4630853d32": {"quality": 0.6390243902439025, "cost": 0.01612172, "time": 131.57029628753662}, "46475b9e75": {"quality": 0.7159001161440186, "cost": 0.011104722, "time": 122.19585280418396}, "46654a1f32": {"quality": 0.6195121951219512, "cost": 0.016634112, "time": 77.13833050727844}, "466a3036b2": {"quality": 0.6498257839721253, "cost": 0.018471648, "time": 207.47669353485108}, "466d4d16dd": {"quality": 0.5902439024390244, "cost": 0.011955879999999999, "time": 118.00577793121337}, "46ed68152d": {"quality": 0.5575493612078979, "cost": 0.016878404, "time": 116.81743607521057}, "476a12876c": {"quality": 0.6195121951219512, "cost": 0.007925976, "time": 132.9657735824585}, "4778401a7a": {"quality": 0.6119047619047618, "cost": 0.036857304, "time": 163.85823788642884}, "47f9115b26": {"quality": 0.6195121951219512, "cost": 0.06548748400000001, "time": 177.22612433433534}, "48043e2304": {"quality": 0.4546341463414635, "cost": 0.013231672000000003, "time": 139.5205948829651}, "487f30e740": {"quality": 0.6455284552845528, "cost": 0.04562078000000001, "time": 155.9966438770294}, "488645cbd9": {"quality": 0.6195121951219512, "cost": 0.0027527399999999996, "time": 95.60826187133789}, "48bf87f7fe": {"quality": 0.4546922183507549, "cost": 0.095429448, "time": 165.5019425392151}, "4909061216": {"quality": 0.2585365853658536, "cost": 0.018523724, "time": 212.3366045475006}, "49731b1ccd": {"quality": 0.2308246225319396, "cost": 0.010751964, "time": 158.96827306747437}, "49ad844bd2": {"quality": 0.6195121951219512, "cost": 0.015603239999999999, "time": 223.71964612007142}, "49ca727e49": {"quality": 0.653739837398374, "cost": 0.0016870099999999999, "time": 64.0728322505951}, "4a23d8eff7": {"quality": 0.6195121951219512, "cost": 0.06434071200000001, "time": 202.87527737617492}, "4a555da784": {"quality": 0.6274796747967479, "cost": 0.06717144000000001, "time": 190.02644109725952}, "4a5cea8b85": {"quality": 0.25497096399535424, "cost": 0.08279934000000001, "time": 153.70768852233886}, "4a767339bd": {"quality": 0.6195121951219512, "cost": 0.065784288, "time": 185.730606508255}, "4aafd39d76": {"quality": 0.6487804878048781, "cost": 0.011711388, "time": 195.04439516067504}, "4aca6e5216": {"quality": 0.4877235772357723, "cost": 0.029520160000000007, "time": 136.51992263793946}, "4b18a647d6": {"quality": 0.7812195121951219, "cost": 0.09365719600000001, "time": 201.93038654327393}, "4bc4528402": {"quality": 0.6845528455284553, "cost": 0.00974208, "time": 175.54990649223328}, "4c158a1a4a": {"quality": 0.47682926829268296, "cost": 0.010284996, "time": 130.92566409111024}, "4c954323e3": {"quality": 0.6668757259001161, "cost": 0.016642032, "time": 211.33878846168517}, "4d91e8a27b": {"quality": 0.38890824622531933, "cost": 0.014601055999999998, "time": 149.77316427230835}, "4dc185389a": {"quality": 0.6353658536585366, "cost": 0.063264392, "time": 190.5721879005432}, "4dd3635bc3": {"quality": 0.6648548199767712, "cost": 0.026166204, "time": 156.0182451725006}, "4dd98ef398": {"quality": 0.48878048780487804, "cost": 0.06819976000000001, "time": 104.27807059288025}, "4dfacd0007": {"quality": 0.8142973286875724, "cost": 0.037908216, "time": 182.91077904701234}, "4e298ee0d4": {"quality": 0.5714634146341463, "cost": 0.012921647999999999, "time": 178.78159699440002}, "4e3443a0f9": {"quality": 0.6566550522648084, "cost": 0.022017672, "time": 196.2869851589203}, "4e4b9db2b8": {"quality": 0.3020325203252033, "cost": 0.010630452000000002, "time": 134.6365571498871}, "4e6509f614": {"quality": 0.5430894308943089, "cost": 0.0084714, "time": 103.83811821937562}, "4e6a83e751": {"quality": 0.4271544715447154, "cost": 0.014789456000000001, "time": 1661.4491950511933}, "4e79c8947f": {"quality": 0.6573170731707317, "cost": 0.016321300000000004, "time": 72.1683876991272}, "4e9504432b": {"quality": 0.667479674796748, "cost": 0.07073244000000001, "time": 194.78891081809996}, "4e962170dc": {"quality": 0.6195121951219512, "cost": 0.06384445600000002, "time": 125.06455044746399}, "4eb0826f21": {"quality": 0.6195121951219512, "cost": 0.05706447600000001, "time": 84.61785073280335}, "4ed41bf2e4": {"quality": 0.5291056910569105, "cost": 0.079910268, "time": 152.21971316337584}, "4f78672528": {"quality": 0.7267711962833914, "cost": 0.073045852, "time": 149.74157452583313}, "4f8cca1195": {"quality": 0.7037514518002322, "cost": 0.021906119999999998, "time": 222.39478406906127}, "500860eaa2": {"quality": 0.6195121951219512, "cost": 0.14549248799999998, "time": 2286.8156877040865}, "50701b505e": {"quality": 0.6195121951219512, "cost": 0.01093236, "time": 183.55579237937928}, "51583a901c": {"quality": 0.5827526132404182, "cost": 0.020865352, "time": 207.0611351966858}, "51aeaf9f3e": {"quality": 0.6361788617886179, "cost": 0.02071614, "time": 60.56362895965576}, "520b52b64c": {"quality": 0.6005458768873403, "cost": 0.07544150000000002, "time": 98.03264541625977}, "521314dab6": {"quality": 0.5414634146341464, "cost": 0.011220016000000001, "time": 111.50055747032165}, "5226eb7ff6": {"quality": 0.7410569105691057, "cost": 0.028896044, "time": 154.04090991020203}, "526878b5eb": {"quality": 0.47317073170731705, "cost": 0.016668683999999996, "time": 211.7058870792389}, "52c1cba6ce": {"quality": 0.6341463414634146, "cost": 0.01895658, "time": 214.20635576248168}, "52f041a70e": {"quality": 0.715691056910569, "cost": 0.00206075, "time": 69.41744899749756}, "533867574b": {"quality": 0.6173054587688733, "cost": 0.06281091200000001, "time": 141.25713596343994}, "53869388bb": {"quality": 0.35284552845528455, "cost": 0.011996687999999997, "time": 385.71526923179624}, "53aefd41e4": {"quality": 0.5982229965156796, "cost": 0.03475750400000001, "time": 124.53529238700867}, "53d2932c4f": {"quality": 0.5304065040650406, "cost": 0.012760976000000002, "time": 143.55552654266359}, "54375d3eba": {"quality": 0.6189430894308943, "cost": 0.024152808000000005, "time": 112.89141573905945}, "5474247f91": {"quality": 0.6195121951219512, "cost": 0.012371476000000001, "time": 125.63727469444275}, "54993bc472": {"quality": 0.6341463414634146, "cost": 0.057053648, "time": 83.04090037345887}, "55358f2285": {"quality": 0.6589430894308943, "cost": 0.06846545600000001, "time": 137.55791845321656}, "5569b4f878": {"quality": 0.7922648083623692, "cost": 0.021984700000000003, "time": 105.12538561820983}, "557d2cf7ba": {"quality": 0.6195121951219512, "cost": 0.012524256000000001, "time": 145.9854365348816}, "55c8aa8935": {"quality": 0.40569105691056906, "cost": 0.018154683999999997, "time": 185.0385425567627}, "55e6bf8f14": {"quality": 0.4878048780487805, "cost": 0.06334899600000002, "time": 175.02917833328246}, "56a0660622": {"quality": 0.5082926829268293, "cost": 0.05803248000000001, "time": 128.3964723110199}, "56a29a28c5": {"quality": 0.6195121951219512, "cost": 0.006270012, "time": 96.01242618560791}, "56c4fd5056": {"quality": 0.6195121951219512, "cost": 0.128267136, "time": 190.17179284095764}, "5703697dbd": {"quality": 0.13241579558652733, "cost": 0.009410826, "time": 197.99875745773315}, "5718f2ed80": {"quality": 0.7897909407665504, "cost": 0.0071524019999999995, "time": 187.86544318199157}, "572a02a59a": {"quality": 0.6716260162601626, "cost": 0.006410382000000001, "time": 171.54228215217591}, "5750713a41": {"quality": 0.5406504065040652, "cost": 0.008179103999999998, "time": 157.33118991851808}, "57757ef15e": {"quality": 0.5353658536585366, "cost": 0.011508492, "time": 147.7572299003601}, "579c81bbe0": {"quality": 0.6146341463414634, "cost": 0.007194356000000001, "time": 109.05034627914429}, "57bed1722f": {"quality": 0.6195121951219512, "cost": 0.00790296, "time": 105.67013120651245}, "585ba6d20b": {"quality": 0.6342973286875726, "cost": 0.016752543999999998, "time": 243.40559344291688}, "589267ac64": {"quality": 0.5423112659698026, "cost": 0.01900224, "time": 108.88611903190613}, "589a1cea79": {"quality": 0.6195121951219512, "cost": 0.014395692000000002, "time": 192.94557881355286}, "58ca42839b": {"quality": 0.7067479674796748, "cost": 0.042082800000000004, "time": 164.78914923667907}, "58dc373441": {"quality": 0.48292682926829267, "cost": 0.062875472, "time": 206.20198302268983}, "59006532b4": {"quality": 0.4398373983739837, "cost": 0.01421172, "time": 209.91840863227844}, "59326c4e00": {"quality": 0.6195121951219512, "cost": 0.009057812, "time": 105.74859399795533}, "593975c75b": {"quality": 0.688780487804878, "cost": 0.09570490400000001, "time": 229.09487490653993}, "596b4f8694": {"quality": 0.4878048780487805, "cost": 0.066254572, "time": 204.71900162696838}, "5971ba4e0d": {"quality": 0.46154471544715453, "cost": 0.02107208, "time": 666.5095714569092}, "5996465c0a": {"quality": 0.7142276422764228, "cost": 0.0669193, "time": 276.37003965377806}, "59d70b9f65": {"quality": 0.7408943089430894, "cost": 0.06718589000000001, "time": 181.75046825408936}, "59f887b67c": {"quality": 0.6195121951219512, "cost": 0.015060804, "time": 185.60658679008483}, "5a22920db4": {"quality": 0.7369802555168409, "cost": 0.004994448, "time": 107.91197681427002}, "5a35020d45": {"quality": 0.6195121951219512, "cost": 0.012432768, "time": 174.93387031555176}, "5aa43da1fc": {"quality": 0.6634959349593497, "cost": 0.04908599600000001, "time": 142.87173461914062}, "5ae0d88127": {"quality": 0.6195121951219512, "cost": 0.07555794, "time": 205.7282470703125}, "5b10fbdbe1": {"quality": 0.443089430894309, "cost": 0.022101736000000004, "time": 206.7174467563629}, "5bade9eb85": {"quality": 0.686225319396051, "cost": 0.014263044000000002, "time": 240.30284028053285}, "5be16744bf": {"quality": 0.40822299651567945, "cost": 0.020666648000000003, "time": 183.62090783119203}, "5c5055e252": {"quality": 0.6341463414634146, "cost": 0.057907604, "time": 170.79225449562074}, "5c53feccd9": {"quality": 0.535191637630662, "cost": 0.022622808, "time": 214.88010306358336}, "5c77c7c2b2": {"quality": 0.486829268292683, "cost": 0.01199347, "time": 164.7836329460144}, "5d298b5b48": {"quality": 0.5081300813008129, "cost": 0.07153200800000001, "time": 211.4806851863861}, "5d41515d2e": {"quality": 0.6988037166085945, "cost": 0.041189079999999996, "time": 152.06495885849}, "5d4babc723": {"quality": 0.2809291521486643, "cost": 0.06604162400000001, "time": 172.66046080589294}, "5d79b50feb": {"quality": 0.3548780487804878, "cost": 0.018818415999999998, "time": 109.70083494186402}, "5dc216cd6b": {"quality": 0.6195121951219512, "cost": 0.014456452, "time": 143.22672152519226}, "5dd68c1b8f": {"quality": 0.6670731707317074, "cost": 0.002938692, "time": 95.08386464118958}, "5de4a882c1": {"quality": 0.3775609756097561, "cost": 0.018079487999999998, "time": 163.3632432937622}, "5e04e1c72d": {"quality": 0.5864227642276423, "cost": 0.060815928000000005, "time": 139.02201280593872}, "5e923cee9e": {"quality": 0.7142740998838559, "cost": 0.037600364, "time": 188.52763104438782}, "5ea2fab380": {"quality": 0.5060975609756098, "cost": 0.006009479999999999, "time": 89.62659049034119}, "5eb3bb525b": {"quality": 0.6526829268292682, "cost": 0.012761808000000003, "time": 132.20405316352844}, "5eea899380": {"quality": 0.4015679442508711, "cost": 0.07097477600000002, "time": 210.64328713417052}, "5f37b3902b": {"quality": 0.6195121951219512, "cost": 0.010051332, "time": 109.50351824760438}, "5f9282df3c": {"quality": 0.37855981416957035, "cost": 0.040870088000000006, "time": 185.25703506469728}, "6019884cf3": {"quality": 0.698931475029036, "cost": 0.06932, "time": 240.45240106582642}, "606352363e": {"quality": 0.19362369337979093, "cost": 0.070268456, "time": 234.0612476825714}, "608728f868": {"quality": 0.6, "cost": 0.068669716, "time": 218.80884642601012}, "60b9e936f1": {"quality": 0.6195121951219512, "cost": 0.056154416000000006, "time": 87.54008474349976}, "60cb623c53": {"quality": 0.46341463414634154, "cost": 0.004160484000000001, "time": 67.07470922470092}, "6234de86b4": {"quality": 0.680650406504065, "cost": 0.08330867000000002, "time": 161.33356328010558}, "62352c6854": {"quality": 0.347479674796748, "cost": 0.07185688, "time": 218.85872540473937}, "63a0aaebed": {"quality": 0.6195121951219512, "cost": 0.005112576000000001, "time": 118.10645513534546}, "647dda686f": {"quality": 0.740650406504065, "cost": 0.038015924, "time": 197.1805561542511}, "6511b21ded": {"quality": 0.5281068524970964, "cost": 0.036544824, "time": 129.3101086616516}, "652c0f4bdf": {"quality": 0.6195121951219512, "cost": 0.01854406, "time": 200.11599526405334}, "6533c85913": {"quality": 0.671869918699187, "cost": 0.016586988, "time": 216.59043788909912}, "65627426e0": {"quality": 0.5750058072009292, "cost": 0.011773232000000003, "time": 1798.7506415367127}, "65b76da9c6": {"quality": 0.6195121951219512, "cost": 0.01047374, "time": 144.71517310142517}, "65be1c1306": {"quality": 0.4783739837398374, "cost": 0.044860399999999995, "time": 1590.2891113758087}, "65e0216208": {"quality": 0.6195121951219512, "cost": 0.01263888, "time": 234.11023540496825}, "65eee615d7": {"quality": 0.6295934959349594, "cost": 0.0011287079999999998, "time": 67.5095160961151}, "6623d7a5ac": {"quality": 0.7706504065040651, "cost": 0.067642702, "time": 238.91902923583984}, "66750c0934": {"quality": 0.7448896631823461, "cost": 0.013174562, "time": 184.41564054489135}, "66776ec181": {"quality": 0.5121951219512195, "cost": 0.014528544, "time": 218.20506610870362}, "66e5ae0a21": {"quality": 0.6195121951219512, "cost": 0.0327125, "time": 125.57788677215576}, "6750a8d7a7": {"quality": 0.7970847851335656, "cost": 0.03363056, "time": 169.0002564907074}, "67632141f6": {"quality": 0.6195121951219512, "cost": 0.007475844000000001, "time": 102.3034242630005}, "67868fcff6": {"quality": 0.44991869918699184, "cost": 0.019169336000000002, "time": 185.75657877922058}, "67bab6732d": {"quality": 0.38455284552845526, "cost": 0.016392688, "time": 159.32311158180238}, "67fe399cf1": {"quality": 0.624390243902439, "cost": 0.010455704, "time": 154.35488867759705}, "6846bd8fb3": {"quality": 0.3334146341463414, "cost": 0.009431328, "time": 157.63436169624327}, "68b4cc3e39": {"quality": 0.44522648083623695, "cost": 0.09505392000000001, "time": 195.20148849487305}, "69a029ae36": {"quality": 0.6195121951219512, "cost": 0.042185764, "time": 178.9181830883026}, "69b3b67de6": {"quality": 0.5991869918699188, "cost": 0.016261864, "time": 182.98831782341003}, "69bf3f6ba0": {"quality": 0.5317073170731708, "cost": 0.012870332000000002, "time": 144.30875706672668}, "6a022c3f73": {"quality": 0.6390243902439025, "cost": 0.006610608, "time": 128.81844487190247}, "6a10c53ad8": {"quality": 0.7610220673635307, "cost": 0.018649594, "time": 219.1147134780884}, "6a6348f69d": {"quality": 0.4230894308943089, "cost": 0.014753328, "time": 1653.7405955314637}, "6a74a11bee": {"quality": 0.5999303135888503, "cost": 0.06506421600000001, "time": 158.56066660881044}, "6a90ec29dd": {"quality": 0.26048780487804873, "cost": 0.032533988, "time": 121.64445943832398}, "6aac59742a": {"quality": 0.6195121951219512, "cost": 0.015229139999999999, "time": 210.34569411277772}, "6b0862c597": {"quality": 0.6967479674796748, "cost": 0.061550596, "time": 178.11355109214782}, "6b3c16def2": {"quality": 0.46341463414634154, "cost": 0.002938128, "time": 90.25655012130737}, "6b99b0e901": {"quality": 0.5002439024390244, "cost": 0.036148564, "time": 119.39121503829956}, "6b9b2b3515": {"quality": 0.3779326364692218, "cost": 0.088494424, "time": 215.23190116882324}, "6bcc02962b": {"quality": 0.6769105691056911, "cost": 0.050208152000000006, "time": 122.08708581924438}, "6c1987a9e3": {"quality": 0.5414634146341464, "cost": 0.06623874400000002, "time": 188.0838879108429}, "6c50123ee1": {"quality": 0.5729268292682927, "cost": 0.05634123, "time": 96.05054454803467}, "6c67c36480": {"quality": 0.6195121951219512, "cost": 0.066650968, "time": 282.4478175640106}, "6c9b9f1363": {"quality": 0.5295121951219512, "cost": 0.06645499, "time": 267.66957621574403}, "6cc813aa68": {"quality": 0.6195121951219512, "cost": 0.008967600000000001, "time": 110.58158135414124}, "6d20c6ace0": {"quality": 0.5440301974448316, "cost": 0.043662036, "time": 164.7413963794708}, "6d444fe21a": {"quality": 0.43720092915214864, "cost": 0.028377092000000003, "time": 181.5676950931549}, "6db70dc3b6": {"quality": 0.5349593495934959, "cost": 0.018471248000000003, "time": 1839.0920428276063}, "6e0690f576": {"quality": 0.6195121951219512, "cost": 0.003072384, "time": 123.15826468467712}, "6e06cc804f": {"quality": 0.7730894308943089, "cost": 0.122657024, "time": 161.19403052330017}, "6e24048a2e": {"quality": 0.6522764227642277, "cost": 0.032542016, "time": 137.39676880836487}, "6e93514f45": {"quality": 0.6195121951219512, "cost": 0.008316468, "time": 125.78651990890503}, "6eae47102b": {"quality": 0.4591869918699187, "cost": 0.012840492, "time": 157.67578744888306}, "6ed4cae469": {"quality": 0.6234959349593496, "cost": 0.021061364000000003, "time": 89.84902305603028}, "6ef3b7127e": {"quality": 0.5697560975609757, "cost": 0.012670767999999999, "time": 80.09802565574645}, "6f60a05c33": {"quality": 0.6195121951219512, "cost": 0.010392396, "time": 162.94261040687562}, "6fe0b3f929": {"quality": 0.6195121951219512, "cost": 0.0075620999999999996, "time": 134.45373644828797}, "700ab1d309": {"quality": 0.6195121951219512, "cost": 0.02821224, "time": 147.49021158218383}, "7040e83d52": {"quality": 0.5623693379790942, "cost": 0.040416854, "time": 170.36972126960754}, "7046765af8": {"quality": 0.200801393728223, "cost": 0.06678788599999999, "time": 205.17822675704957}, "70b7c92ce8": {"quality": 0.6439024390243903, "cost": 0.06304620400000002, "time": 216.9712851047516}, "70c850e039": {"quality": 0.6195121951219512, "cost": 0.00682044, "time": 111.72890758514404}, "7112a7e64c": {"quality": 0.5082113821138211, "cost": 0.008551890000000001, "time": 102.93057270050049}, "71b615468b": {"quality": 0.31138211382113823, "cost": 0.016398592, "time": 133.36204581260682}, "723fd5589a": {"quality": 0.6195121951219512, "cost": 0.025608394, "time": 148.1404324531555}, "7250da0f41": {"quality": 0.6195121951219512, "cost": 0.010252872, "time": 102.41908440589904}, "7274a50778": {"quality": 0.5536585365853659, "cost": 0.06315958, "time": 127.02501587867737}, "72d022ce33": {"quality": 0.8057259001161441, "cost": 0.08692716000000002, "time": 196.42877383232116}, "7347cf0308": {"quality": 0.6195121951219512, "cost": 0.007518113999999999, "time": 112.80636467933655}, "736e652158": {"quality": 0.6195121951219512, "cost": 0.010933332, "time": 202.134419298172}, "739b1f81dc": {"quality": 0.7352845528455284, "cost": 0.005161632, "time": 102.21428322792053}, "742a5c0552": {"quality": 0.6585365853658537, "cost": 0.05980691200000001, "time": 95.91068015098571}, "742ec1b2e1": {"quality": 0.691869918699187, "cost": 0.06350464000000001, "time": 189.14654784202577}, "7435fd54f8": {"quality": 0.6567131242741, "cost": 0.06201505600000001, "time": 203.98798232078553}, "7445d99939": {"quality": 0.4034843205574912, "cost": 0.011135484, "time": 153.3177831172943}, "7466a5f424": {"quality": 0.4263414634146342, "cost": 0.0244955, "time": 218.84586930274963}, "74a0be215b": {"quality": 0.48292682926829267, "cost": 0.06485306800000001, "time": 214.73199005126952}, "74d7f64b8c": {"quality": 0.6195121951219512, "cost": 0.011531088000000002, "time": 211.1984474658966}, "7524905580": {"quality": 0.7694889663182346, "cost": 0.012599868, "time": 157.22432436943055}, "752d9649f2": {"quality": 0.5467479674796748, "cost": 0.022367612000000002, "time": 204.80810337066652}, "75d61c2cd0": {"quality": 0.7049128919860628, "cost": 0.062316, "time": 160.31273555755615}, "7604c0aa13": {"quality": 0.6195121951219512, "cost": 0.0142076, "time": 142.32078042030335}, "76c09db721": {"quality": 0.7820441347270615, "cost": 0.04096148000000001, "time": 236.8196849822998}, "774f268b66": {"quality": 0.6810569105691057, "cost": 0.08915512, "time": 201.10393743515016}, "7765576286": {"quality": 0.567479674796748, "cost": 0.016253124, "time": 222.17675213813783}, "77c02b00c1": {"quality": 0.5561091753774681, "cost": 0.017938308, "time": 201.10154690742493}, "7801da66b9": {"quality": 0.6195121951219512, "cost": 0.00746667, "time": 114.64372763633727}, "782d52674e": {"quality": 0.3201974448315911, "cost": 0.06483101399999999, "time": 193.24506635665892}, "786e5d0af5": {"quality": 0.6195121951219512, "cost": 0.016454096, "time": 197.87756876945497}, "7878563d63": {"quality": 0.1632171893147503, "cost": 0.028198652, "time": 84.60407543182373}, "79e1ca9b3c": {"quality": 0.6195121951219512, "cost": 0.06867538, "time": 110.48155570030212}, "79fad58f07": {"quality": 0.46674796747967484, "cost": 0.029654968000000004, "time": 679.6173066616059}, "7a207b42a8": {"quality": 0.6195121951219512, "cost": 0.012325835999999998, "time": 163.009268951416}, "7a2cdc546c": {"quality": 0.2631707317073171, "cost": 0.06310833, "time": 160.65065565109253}, "7a42a77788": {"quality": 0.6195121951219512, "cost": 0.06019363200000001, "time": 138.42030897140503}, "7a58d3472b": {"quality": 0.5959349593495935, "cost": 0.025425438, "time": 147.28676762580872}, "7b3937c1f1": {"quality": 0.39544715447154466, "cost": 0.02868438, "time": 89.45209522247315}, "7b6f44618e": {"quality": 0.5699186991869919, "cost": 0.018827976000000003, "time": 179.5670045375824}, "7c62576527": {"quality": 0.5152845528455284, "cost": 0.042298876, "time": 167.9848653316498}, "7c89a2b69e": {"quality": 0.6195121951219512, "cost": 0.019231344, "time": 252.72414374351501}, "7c96c9712f": {"quality": 0.6536585365853659, "cost": 0.06122004000000001, "time": 127.95261840820312}, "7ca066aa1c": {"quality": 0.7818002322880371, "cost": 0.040026332000000005, "time": 187.7031367778778}, "7cb5591f27": {"quality": 0.3416840882694541, "cost": 0.0826827, "time": 138.52483682632447}, "7cf56a7fbc": {"quality": 0.6195121951219512, "cost": 0.08315224400000001, "time": 106.8193588256836}, "7d44f0959d": {"quality": 0.46680603948896626, "cost": 0.011511138, "time": 133.2327687740326}, "7d60c38c5c": {"quality": 0.7220441347270616, "cost": 0.00538134, "time": 131.21526570320128}, "7d9b4535ac": {"quality": 0.6769802555168408, "cost": 0.063389518, "time": 211.52207164764405}, "7daf7ff182": {"quality": 0.5284552845528456, "cost": 0.017232744, "time": 170.8186996459961}, "7dcedb3d02": {"quality": 0.6593960511033682, "cost": 0.06829533600000001, "time": 231.27069878578186}, "7e22f12cd1": {"quality": 0.5735540069686411, "cost": 0.03302558, "time": 186.45001711845399}, "7e53a50b13": {"quality": 0.4565737514518003, "cost": 0.033876876, "time": 151.98724694252013}, "7ed07ad40a": {"quality": 0.6177700348432056, "cost": 0.0116127, "time": 211.14939546585083}, "7fa67a7656": {"quality": 0.5077235772357723, "cost": 0.01125696, "time": 107.8444951057434}, "7fc6c84bdf": {"quality": 0.7865969802555168, "cost": 0.03407424, "time": 176.73210568428038}, "7ff8a779cc": {"quality": 0.7043089430894309, "cost": 0.06711379800000002, "time": 168.2764458656311}, "801af99400": {"quality": 0.7181765389082462, "cost": 0.07170897000000001, "time": 184.03885922431945}, "806881adcb": {"quality": 0.6209175377468059, "cost": 0.012441478, "time": 130.4481972694397}, "80ad4122e4": {"quality": 0.6211382113821139, "cost": 0.07175158800000002, "time": 199.85183753967286}, "80be7df955": {"quality": 0.6195121951219512, "cost": 0.0029887200000000003, "time": 99.58167638778687}, "80bf60c422": {"quality": 0.6195121951219512, "cost": 0.005357952000000001, "time": 138.18060364723206}, "81333c7a33": {"quality": 0.5926829268292683, "cost": 0.027174752000000003, "time": 143.90170640945433}, "813e75210b": {"quality": 0.3377816492450639, "cost": 0.05362793600000001, "time": 1273.6602407932282}, "81660ae8b2": {"quality": 0.6195121951219512, "cost": 0.06746300400000001, "time": 194.1754295349121}, "816958b5d1": {"quality": 0.6963414634146342, "cost": 0.07060810400000002, "time": 230.40107922554017}, "81ab2ef3f4": {"quality": 0.3821138211382114, "cost": 0.030218416000000005, "time": 135.14831619262696}, "829df73946": {"quality": 0.6964227642276423, "cost": 0.008790552, "time": 111.97322883605958}, "82ea1bd1b9": {"quality": 0.8085365853658537, "cost": 0.069294098, "time": 143.64819779396058}, "8357183895": {"quality": 0.6195121951219512, "cost": 0.014406096000000002, "time": 247.72333025932312}, "8392a6083a": {"quality": 0.6195121951219512, "cost": 0.017490875999999995, "time": 201.15325956344606}, "83aee532b7": {"quality": 0.6195121951219512, "cost": 0.064240072, "time": 249.5134038925171}, "83b26646c3": {"quality": 0.6652845528455285, "cost": 0.04101521600000001, "time": 139.2657244682312}, "83c9e66ec6": {"quality": 0.6439024390243903, "cost": 0.012396760000000001, "time": 179.99957365989684}, "847a0e5db5": {"quality": 0.6909756097560976, "cost": 0.06664256800000001, "time": 183.08391642570496}, "847fd49235": {"quality": 0.3338327526132404, "cost": 0.006645311999999999, "time": 167.98347339630126}, "849100224d": {"quality": 0.3345528455284553, "cost": 0.08884810000000001, "time": 220.43858489990234}, "84b91c37ab": {"quality": 0.24628339140534267, "cost": 0.069210264, "time": 236.29133720397948}, "8519bef585": {"quality": 0.6195121951219512, "cost": 0.005065452, "time": 122.15238361358642}, "85c94a5505": {"quality": 0.4611382113821138, "cost": 0.056482279999999996, "time": 1274.952497625351}, "85eda38404": {"quality": 0.6357142857142857, "cost": 0.01686636, "time": 103.26268811225891}, "862183bfb9": {"quality": 0.6951219512195121, "cost": 0.012958452, "time": 162.91549973487855}, "8631e49c94": {"quality": 0.6439024390243903, "cost": 0.06574006400000001, "time": 227.21288151741027}, "8668f65f05": {"quality": 0.6195121951219512, "cost": 0.01596044, "time": 191.07359557151796}, "86bf6375af": {"quality": 0.6560975609756097, "cost": 0.06964888, "time": 260.4157979488373}, "870e2f87b4": {"quality": 0.7660046457607433, "cost": 0.025822204000000005, "time": 201.3596661567688}, "887ad124e1": {"quality": 0.709349593495935, "cost": 0.0070279439999999995, "time": 150.77690081596376}, "8886cb3082": {"quality": 0.6834843205574914, "cost": 0.019317688000000003, "time": 211.99706134796142}, "88e71efa9b": {"quality": 0.28095238095238095, "cost": 0.046789276000000005, "time": 186.75326628684996}, "8941621423": {"quality": 0.6786991869918699, "cost": 0.012530784, "time": 167.57245116233827}, "8950a6efe0": {"quality": 0.48292682926829267, "cost": 0.05844699200000001, "time": 136.73901495933532}, "8961e4d901": {"quality": 0.6195121951219512, "cost": 0.010754568, "time": 156.90380158424378}, "8974aa89a0": {"quality": 0.5727642276422764, "cost": 0.01088668, "time": 377.91359758377075}, "89836d2020": {"quality": 0.6195121951219512, "cost": 0.06319070400000001, "time": 132.40995478630066}, "89a289907e": {"quality": 0.7753426248548199, "cost": 0.014910682, "time": 212.81768522262573}, "89a35a09b1": {"quality": 0.525609756097561, "cost": 0.011930976, "time": 194.43942880630493}, "8a3a35c762": {"quality": 0.6390243902439025, "cost": 0.07371796400000001, "time": 188.05857005119324}, "8a50695d1f": {"quality": 0.5045644599303136, "cost": 0.027424088000000003, "time": 111.09706916809083}, "8ab351aa13": {"quality": 0.6526829268292682, "cost": 0.089840192, "time": 137.60196342468262}, "8ac8b5773a": {"quality": 0.6292682926829268, "cost": 0.06544508800000001, "time": 216.46768646240236}, "8acd758b7f": {"quality": 0.6600929152148665, "cost": 0.008952419999999999, "time": 151.0333396911621}, "8b10891ea5": {"quality": 0.4276422764227642, "cost": 0.07350195200000001, "time": 188.96431422233582}, "8b721bbc6f": {"quality": 0.6195121951219512, "cost": 0.012898752, "time": 215.82190022468566}, "8b77535cce": {"quality": 0.7979558652729384, "cost": 0.07287564, "time": 215.97581152915956}, "8bbbe0f52a": {"quality": 0.6195121951219512, "cost": 0.008104248, "time": 149.25933980941772}, "8bc184f385": {"quality": 0.37040650406504066, "cost": 0.008849639999999999, "time": 130.85588617324828}, "8bf5c3eadc": {"quality": 0.6195121951219512, "cost": 0.008037444, "time": 130.57925519943237}, "8bf80a50cb": {"quality": 0.45910569105691057, "cost": 0.06549705000000001, "time": 191.59308562278747}, "8c274ca255": {"quality": 0.7277932636469222, "cost": 0.07598581600000001, "time": 203.60676689147948}, "8d7594020b": {"quality": 0.6195121951219512, "cost": 0.06030525600000001, "time": 142.03003664016722}, "8d79e03266": {"quality": 0.728513356562137, "cost": 0.019196192, "time": 166.50991864204406}, "8d90814b94": {"quality": 0.35382113821138217, "cost": 0.019129876, "time": 218.55778388977052}, "8e1a01da19": {"quality": 0.6195121951219512, "cost": 0.06321022000000001, "time": 206.71381578445434}, "8e2498635d": {"quality": 0.8135772357723576, "cost": 0.142084164, "time": 153.71730332374574}, "8e5842ccbd": {"quality": 0.7329268292682927, "cost": 0.007325802000000001, "time": 112.95453724861144}, "8e5daf241e": {"quality": 0.6923228803716608, "cost": 0.060446860000000005, "time": 186.30223965644836}, "8e9715ee01": {"quality": 0.5639837398373984, "cost": 0.08655280400000002, "time": 139.08190727233887}, "8e9b7300d4": {"quality": 0.4639837398373984, "cost": 0.06058211200000001, "time": 150.15705633163452}, "8f29fab8ac": {"quality": 0.4739140534262486, "cost": 0.06764498000000001, "time": 200.2176959514618}, "8f44d89429": {"quality": 0.7065040650406504, "cost": 0.069996624, "time": 185.83366765975953}, "8f4caddfe6": {"quality": 0.5421138211382114, "cost": 0.021058936, "time": 161.7523732662201}, "8f4edde3f0": {"quality": 0.5726480836236936, "cost": 0.03185762, "time": 159.63181648254394}, "8f9cefbc22": {"quality": 0.6585365853658537, "cost": 0.05867756800000001, "time": 105.53310813903809}, "9025e2480f": {"quality": 0.6595934959349593, "cost": 0.00965967, "time": 128.92424964904785}, "9028588af4": {"quality": 0.35292682926829266, "cost": 0.09341379200000001, "time": 1090.4150963783263}, "9059fd80ad": {"quality": 0.5902439024390244, "cost": 0.027686471999999997, "time": 2047.5434857845307}, "90d5e40c1b": {"quality": 0.6195121951219512, "cost": 0.007589304, "time": 120.88176670074463}, "90d9a86a2a": {"quality": 0.6195121951219512, "cost": 0.0036013499999999997, "time": 98.5603612422943}, "90ff13783c": {"quality": 0.6255284552845529, "cost": 0.018824976, "time": 219.693346452713}, "90ff8eb055": {"quality": 0.5428571428571429, "cost": 0.015549384, "time": 206.51184477806092}, "9104e31369": {"quality": 0.6640650406504065, "cost": 0.022935780000000003, "time": 229.26985802650452}, "918983323f": {"quality": 0.5085365853658537, "cost": 0.045187920000000006, "time": 1981.811208820343}, "91928dfdd9": {"quality": 0.6747967479674797, "cost": 0.06523348, "time": 182.1119598388672}, "91c800af6b": {"quality": 0.6161904761904762, "cost": 0.018671568000000003, "time": 212.29187331199645}, "91e841cfd5": {"quality": 0.6195121951219512, "cost": 0.067784204, "time": 200.92469301223755}, "9253901a1f": {"quality": 0.6195121951219512, "cost": 0.07052953600000002, "time": 216.47048025131227}, "9288642e53": {"quality": 0.6195121951219512, "cost": 0.008932916, "time": 94.32965250015259}, "92ba9c5be3": {"quality": 0.567479674796748, "cost": 0.064902632, "time": 138.75575132369994}, "92c9dcd43b": {"quality": 0.6195121951219512, "cost": 0.014990692, "time": 237.96368551254272}, "93011c0821": {"quality": 0.6195121951219512, "cost": 0.019036675999999995, "time": 267.35766806602476}, "933b4d17dd": {"quality": 0.5723577235772358, "cost": 0.012143372, "time": 172.0639699459076}, "9373267bdb": {"quality": 0.5410452961672474, "cost": 0.06831846000000001, "time": 258.8440625667572}, "94010928c6": {"quality": 0.25325203252032524, "cost": 0.011834136, "time": 131.6460223197937}, "9403809e44": {"quality": 0.4588617886178861, "cost": 0.016092682, "time": 231.8062086582184}, "940c88ddc5": {"quality": 0.5727874564459932, "cost": 0.029609188, "time": 126.90794463157654}, "948f4081ba": {"quality": 0.6792102206736355, "cost": 0.039854912000000006, "time": 248.46457772254945}, "94ac356663": {"quality": 0.5626016260162602, "cost": 0.012098179999999998, "time": 162.48227286338806}, "94dff9a424": {"quality": 0.7495934959349594, "cost": 0.013244731999999999, "time": 187.24048733711243}, "9508356a2e": {"quality": 0.6195121951219512, "cost": 0.012070511999999999, "time": 184.9062297821045}, "9539d0e28c": {"quality": 0.37120789779326363, "cost": 0.04079825200000001, "time": 198.3246217250824}, "956bdcc254": {"quality": 0.12377468060394889, "cost": 0.026780572000000002, "time": 157.82376141548156}, "957d0dafc5": {"quality": 0.5816260162601626, "cost": 0.067579808, "time": 153.21978812217714}, "9594b0c783": {"quality": 0.6195121951219512, "cost": 0.006768972, "time": 171.20207090377806}, "95a7b80c2a": {"quality": 0.797979094076655, "cost": 0.02950122, "time": 181.7308371067047}, "964c671f18": {"quality": 0.7772822299651567, "cost": 0.021085212, "time": 212.14437518119811}, "9679fe2b69": {"quality": 0.6195121951219512, "cost": 0.007065155999999999, "time": 184.30773782730103}, "968fc95038": {"quality": 0.4525319396051103, "cost": 0.005385312, "time": 147.3559940338135}, "96b487c724": {"quality": 0.440650406504065, "cost": 0.0038495599999999993, "time": 780.6981408596039}, "96c30205f5": {"quality": 0.5457723577235772, "cost": 0.064298388, "time": 146.8725693702698}, "96f87d6483": {"quality": 0.7184436701509872, "cost": 0.021446996000000003, "time": 236.88481707572936}, "972c83b002": {"quality": 0.6195121951219512, "cost": 0.008979192, "time": 162.82133150100708}, "977a4d6b6b": {"quality": 0.28170731707317076, "cost": 0.021416136000000002, "time": 220.9775969028473}, "97bc30bd83": {"quality": 0.21211382113821137, "cost": 0.015064284, "time": 209.8210876464844}, "980db5f95f": {"quality": 0.6791869918699187, "cost": 0.059587308000000005, "time": 168.43710894584655}, "9836765d41": {"quality": 0.6195121951219512, "cost": 0.06211643600000001, "time": 171.59594984054564}, "99569e3937": {"quality": 0.6777932636469222, "cost": 0.07203462400000002, "time": 259.5382764339447}, "99ea16a9a6": {"quality": 0.815528455284553, "cost": 0.034362648, "time": 175.2437418460846}, "9a5b39370f": {"quality": 0.6195121951219512, "cost": 0.068652448, "time": 237.78598909378053}, "9aa4abfb50": {"quality": 0.6195121951219512, "cost": 0.005156892000000001, "time": 114.41352844238281}, "9ad7a98c31": {"quality": 0.7987108013937282, "cost": 0.07103688, "time": 182.73348231315612}, "9b3fb79bcb": {"quality": 0.7849477351916376, "cost": 0.028800140000000002, "time": 167.58384928703308}, "9b6d4915f3": {"quality": 0.6195121951219512, "cost": 0.0058737600000000004, "time": 96.19086871147155}, "9bae5bafc1": {"quality": 0.4828222996515679, "cost": 0.020766276000000004, "time": 144.2882854938507}, "9be8a5f317": {"quality": 0.6097560975609756, "cost": 0.070895624, "time": 241.7383065700531}, "9c549db0a7": {"quality": 0.6195121951219512, "cost": 0.021638197999999997, "time": 138.09213070869447}, "9c85f8cfcb": {"quality": 0.6195121951219512, "cost": 0.004881552, "time": 85.23935956954956}, "9c8cc46e6c": {"quality": 0.6195121951219512, "cost": 0.004920108, "time": 113.78448638916015}, "9c97d35a30": {"quality": 0.8032520325203251, "cost": 0.011073209999999998, "time": 181.72330527305604}, "9cbe7858a2": {"quality": 0.6195121951219512, "cost": 0.06554662800000002, "time": 227.9875654697418}, "9ce2c3fd98": {"quality": 0.6195121951219512, "cost": 0.006856308, "time": 155.26185359954835}, "9d18cd0737": {"quality": 0.4644018583042973, "cost": 0.006854579999999999, "time": 124.20924863815307}, "9e06360bc9": {"quality": 0.46796747967479674, "cost": 0.011727056, "time": 160.3625663280487}, "9f07a95e69": {"quality": 0.5894308943089431, "cost": 0.073255632, "time": 234.6284239768982}, "9fb157be35": {"quality": 0.6195121951219512, "cost": 0.012696056, "time": 159.5446131706238}, "a04ac8e33a": {"quality": 0.6195121951219512, "cost": 0.06885804400000001, "time": 157.56720504760742}, "a04bc6e116": {"quality": 0.5231707317073171, "cost": 0.06362432400000001, "time": 166.92331409454346}, "a0b81be5b4": {"quality": 0.3871660859465737, "cost": 0.00347016, "time": 123.85612168312073}, "a0c85d260e": {"quality": 0.69602787456446, "cost": 0.024224592, "time": 235.78984532356262}, "a0dc9f50ac": {"quality": 0.18065040650406505, "cost": 0.007600488, "time": 130.9164544582367}, "a18225b7b5": {"quality": 0.6352032520325204, "cost": 0.050534568, "time": 152.35800580978395}, "a1d822289e": {"quality": 0.4399303135888502, "cost": 0.08478274000000001, "time": 163.93071274757386}, "a25596c056": {"quality": 0.6195121951219512, "cost": 0.067823316, "time": 216.80869884490966}, "a2811c7324": {"quality": 0.6701277584204414, "cost": 0.028778920000000003, "time": 168.49099044799806}, "a2aa082d14": {"quality": 0.6195121951219512, "cost": 0.022896492, "time": 119.0820493221283}, "a2cd339ad9": {"quality": 0.6406504065040651, "cost": 0.018992632, "time": 243.06863555908203}, "a2fd03e6a5": {"quality": 0.47276422764227644, "cost": 0.07033531999999999, "time": 1132.2907946109772}, "a31e87d7cb": {"quality": 0.6329268292682927, "cost": 0.00273726, "time": 105.99368834495544}, "a3e23c327b": {"quality": 0.5317073170731708, "cost": 0.068050688, "time": 204.56698241233826}, "a457f6c300": {"quality": 0.6195121951219512, "cost": 0.012840227999999999, "time": 168.6292993545532}, "a47de025c8": {"quality": 0.6195121951219512, "cost": 0.007961357999999998, "time": 138.18772959709167}, "a515a9c8cc": {"quality": 0.6253658536585366, "cost": 0.005416404, "time": 126.60180377960205}, "a5949b76ec": {"quality": 0.6195121951219512, "cost": 0.003754764, "time": 81.27935070991516}, "a60dd076b8": {"quality": 0.405609756097561, "cost": 0.0034407720000000004, "time": 118.2785505771637}, "a6297a6c56": {"quality": 0.6195121951219512, "cost": 0.005110776, "time": 118.47536420822144}, "a62b7555b9": {"quality": 0.4455284552845528, "cost": 0.07002202800000001, "time": 229.59565677642823}, "a63f48e8ca": {"quality": 0.6195121951219512, "cost": 0.069574668, "time": 132.71091380119324}, "a6460dbb7c": {"quality": 0.7212078977932637, "cost": 0.009528016, "time": 130.55586276054382}, "a66a4cf4b0": {"quality": 0.567479674796748, "cost": 0.07641536000000002, "time": 202.29455227851867}, "a6e2d69222": {"quality": 0.4239024390243902, "cost": 0.045957064, "time": 178.1903151988983}, "a717c4c535": {"quality": 0.6195121951219512, "cost": 0.019736044, "time": 234.4358515739441}, "a76afe9960": {"quality": 0.5696051103368177, "cost": 0.060384544000000005, "time": 172.19008646011352}, "a7a6353090": {"quality": 0.4487224157955866, "cost": 0.066870862, "time": 166.1868350982666}, "a80f6535b1": {"quality": 0.5308130081300814, "cost": 0.020093776, "time": 97.89270992279053}, "a86b137d7f": {"quality": 0.6815447154471544, "cost": 0.006451132, "time": 93.94514923095703}, "a88eb1493c": {"quality": 0.41745644599303144, "cost": 0.0066783, "time": 116.80834813117981}, "a89c533d6c": {"quality": 0.6195121951219512, "cost": 0.06598649999999999, "time": 83.00586166381837}, "a8d8264600": {"quality": 0.4808943089430895, "cost": 0.071912272, "time": 251.74223246574402}, "a8eb36b210": {"quality": 0.4832520325203252, "cost": 0.03806308, "time": 194.64342403411865}, "a95b4a6dd0": {"quality": 0.6195121951219512, "cost": 0.012172187999999999, "time": 172.00175199508666}, "a9621ea4e6": {"quality": 0.47317073170731705, "cost": 0.010803696, "time": 194.9391739845276}, "a9721a0a50": {"quality": 0.6624390243902439, "cost": 0.009840604, "time": 122.32440161705017}, "a972b02c61": {"quality": 0.7189430894308944, "cost": 0.03160314, "time": 67.33741846084595}, "a9c5c4e311": {"quality": 0.7951335656213705, "cost": 0.028237476, "time": 182.70986766815184}, "a9d96670eb": {"quality": 0.6195121951219512, "cost": 0.05767817200000001, "time": 169.8078365802765}, "a9e8c974d3": {"quality": 0.6195121951219512, "cost": 0.06961316799999999, "time": 225.65897822380066}, "aa08180e36": {"quality": 0.6146341463414634, "cost": 0.014778952000000001, "time": 221.92620844841002}, "aa38702a02": {"quality": 0.6146341463414634, "cost": 0.006915852, "time": 164.99627866744996}, "aadbfc418b": {"quality": 0.45479674796747965, "cost": 0.00576612, "time": 161.90984616279601}, "aaeb8b0010": {"quality": 0.6195121951219512, "cost": 0.06474566000000001, "time": 209.02622961997986}, "ab288ee7f2": {"quality": 0.5373983739837398, "cost": 0.06573288000000001, "time": 226.1878888130188}, "ab43b02cb0": {"quality": 0.6195121951219512, "cost": 0.01377632, "time": 126.55581045150757}, "aba1d612cc": {"quality": 0.5083391405342625, "cost": 0.09728845600000001, "time": 219.89660325050355}, "ac208e7a1d": {"quality": 0.4333333333333334, "cost": 0.01616966, "time": 142.93958106040955}, "ac2224adbe": {"quality": 0.6195121951219512, "cost": 0.039715708, "time": 125.78687748908996}, "ac828ffe70": {"quality": 0.5008130081300812, "cost": 0.003011208, "time": 454.9158252716064}, "ac9fdc1550": {"quality": 0.5360162601626015, "cost": 0.026754198, "time": 173.65110726356505}, "aca957ecff": {"quality": 0.8072590011614402, "cost": 0.08830083200000001, "time": 230.63441123962403}, "acfe1ed920": {"quality": 0.35609756097560974, "cost": 0.02273798, "time": 88.96396307945251}, "ad3efe44c3": {"quality": 0.6195121951219512, "cost": 0.004642056, "time": 92.58378911018372}, "ad41c95a99": {"quality": 0.3482113821138212, "cost": 0.045380508, "time": 154.1578179359436}, "ad48432c22": {"quality": 0.6569105691056911, "cost": 0.015225708, "time": 247.30540533065795}, "ad6ebbba8d": {"quality": 0.6195121951219512, "cost": 0.012972292, "time": 209.87320923805237}, "ad90055ef6": {"quality": 0.6195121951219512, "cost": 0.00457182, "time": 93.97476525306702}, "adab1e0fb1": {"quality": 0.46829268292682935, "cost": 0.008422604000000002, "time": 115.4403871536255}, "ae655ec593": {"quality": 0.35599303135888505, "cost": 0.006649944, "time": 167.62679462432862}, "ae94b172be": {"quality": 0.5552613240418119, "cost": 0.013280692, "time": 162.66643962860107}, "aec9dc5873": {"quality": 0.6195121951219512, "cost": 0.055165792000000005, "time": 132.68569073677062}, "af360c323c": {"quality": 0.6195121951219512, "cost": 0.006864167999999999, "time": 195.27329888343812}, "af90567194": {"quality": 0.7732404181184669, "cost": 0.02378152, "time": 264.66726565361023}, "afe77d0f89": {"quality": 0.768513356562137, "cost": 0.069933622, "time": 251.59823207855226}, "b0948c05b6": {"quality": 0.6380487804878049, "cost": 0.017833904, "time": 149.21853432655334}, "b0c4a6640b": {"quality": 0.4317653890824623, "cost": 0.063453446, "time": 221.18421225547792}, "b12caafd58": {"quality": 0.46829268292682935, "cost": 0.06446362400000001, "time": 217.29487829208375}, "b18168b9c1": {"quality": 0.3608130081300814, "cost": 0.010323167999999999, "time": 189.31807408332824}, "b1a7428a01": {"quality": 0.21260162601626015, "cost": 0.06952381600000002, "time": 206.16540212631224}, "b1acdebb48": {"quality": 0.6821138211382114, "cost": 0.102856868, "time": 183.48106842041017}, "b1b06f4ee7": {"quality": 0.6195121951219512, "cost": 0.12662600000000002, "time": 142.81967635154723}, "b1cf8d33e5": {"quality": 0.3186178861788618, "cost": 0.011143266, "time": 175.05015845298766}, "b1e9ab6b1a": {"quality": 0.5263414634146341, "cost": 0.07167981200000001, "time": 206.59926490783693}, "b2b057ba41": {"quality": 0.4799186991869918, "cost": 0.0052038779999999995, "time": 142.83305773735046}, "b2e063499d": {"quality": 0.6911498257839722, "cost": 0.018540096, "time": 176.64938292503356}, "b33412410e": {"quality": 0.6195121951219512, "cost": 0.03299608, "time": 125.82604823112487}, "b3369775dc": {"quality": 0.49406504065040646, "cost": 0.017937791999999998, "time": 202.59222974777222}, "b3b9205f60": {"quality": 0.20133565621370497, "cost": 0.03923410000000001, "time": 184.9780117034912}, "b3c56f0b3c": {"quality": 0.594959349593496, "cost": 0.06256806000000001, "time": 168.73735642433167}, "b3f20b706d": {"quality": 0.6195121951219512, "cost": 0.0037520640000000003, "time": 106.53766713142394}, "b4002173ee": {"quality": 0.5188617886178861, "cost": 0.007600319999999999, "time": 109.14190998077393}, "b46d382384": {"quality": 0.7555284552845528, "cost": 0.12484462, "time": 146.6869523525238}, "b4b2482ef9": {"quality": 0.4543554006968641, "cost": 0.06673528000000001, "time": 163.60684385299683}, "b4be043238": {"quality": 0.3460394889663182, "cost": 0.08778872000000001, "time": 150.07519998550416}, "b531bd0548": {"quality": 0.3177584204413472, "cost": 0.08969832, "time": 202.44645614624022}, "b56c312eda": {"quality": 0.5277351916376307, "cost": 0.016686852, "time": 249.4414801120758}, "b5e2b41c1c": {"quality": 0.551684088269454, "cost": 0.009034968, "time": 169.00915660858155}, "b61ce57a90": {"quality": 0.36904761904761907, "cost": 0.039441384, "time": 192.418292427063}, "b64ddb14f9": {"quality": 0.3847967479674797, "cost": 0.02944528, "time": 824.0557322502136}, "b67107a43e": {"quality": 0.6195121951219512, "cost": 0.01083792, "time": 248.4702594280243}, "b67720aa5c": {"quality": 0.6195121951219512, "cost": 0.05931710800000001, "time": 146.00394463539124}, "b682a23b89": {"quality": 0.7337746806039489, "cost": 0.013234963999999998, "time": 213.58269662857055}, "b69ef5add4": {"quality": 0.6584204413472706, "cost": 0.07431707200000001, "time": 161.14033164978028}, "b796b7ffd3": {"quality": 0.3386178861788618, "cost": 0.014401015999999999, "time": 259.1940643310547}, "b7a0083dc4": {"quality": 0.6195121951219512, "cost": 0.061847876, "time": 214.91841106414796}, "b7d0e8557f": {"quality": 0.5686411149825784, "cost": 0.012575807999999999, "time": 247.72326970100403}, "b8317a3a8c": {"quality": 0.7867131242740998, "cost": 0.06423970400000001, "time": 181.56251420974732}, "b8ab3d2f25": {"quality": 0.5626016260162602, "cost": 0.015544804, "time": 184.08619875907897}, "b8b569172f": {"quality": 0.5490243902439025, "cost": 0.015828372, "time": 178.33178467750548}, "b8f5ab44bb": {"quality": 0.634959349593496, "cost": 0.07826424000000001, "time": 262.68980021476744}, "b91e7fdb29": {"quality": 0.6195121951219512, "cost": 0.012287124, "time": 227.29503026008607}, "b932beaaa6": {"quality": 0.544959349593496, "cost": 0.06720848000000001, "time": 186.7761948108673}, "b9770c2261": {"quality": 0.5645644599303136, "cost": 0.00541122, "time": 150.7996078491211}, "b9bb1e6f8d": {"quality": 0.6998373983739838, "cost": 0.018532544, "time": 244.14531807899476}, "b9d0e8740c": {"quality": 0.6134146341463415, "cost": 0.07111140800000001, "time": 214.80163626670839}, "b9da208432": {"quality": 0.641869918699187, "cost": 0.019034844000000002, "time": 232.56544876098633}, "ba3223f6ac": {"quality": 0.6195121951219512, "cost": 0.005706648, "time": 151.19898543357849}, "bb13365175": {"quality": 0.7763995354239257, "cost": 0.042266292000000004, "time": 224.2499403476715}, "bb1b3a4d29": {"quality": 0.8029384436701509, "cost": 0.064898518, "time": 256.5067723274231}, "bb6536b0ab": {"quality": 0.6195121951219512, "cost": 0.013670112000000002, "time": 159.04916138648986}, "bbba9dd6ae": {"quality": 0.5152032520325204, "cost": 0.008736126, "time": 125.51340088844299}, "bbde69a1ae": {"quality": 0.47317073170731705, "cost": 0.07075216, "time": 211.4701558113098}, "bc29a0c0fe": {"quality": 0.6923809523809524, "cost": 0.08504816000000001, "time": 152.53999376296997}, "bc3d02f753": {"quality": 0.6195121951219512, "cost": 0.0033908220000000004, "time": 108.17700786590576}, "bc4c1fcc64": {"quality": 0.25837398373983744, "cost": 0.008965720000000002, "time": 110.68981971740723}, "bd30d27f62": {"quality": 0.8004529616724738, "cost": 0.13683402800000002, "time": 179.2394115447998}, "bd99b2fb21": {"quality": 0.6292682926829268, "cost": 0.013529848, "time": 165.33139224052428}, "bddc7d2a34": {"quality": 0.7716144018583042, "cost": 0.06608894800000001, "time": 242.8191273212433}, "be2ae88f70": {"quality": 0.6195121951219512, "cost": 0.0070768079999999995, "time": 175.22385454177856}, "be4740f38f": {"quality": 0.5178048780487805, "cost": 0.018557584000000002, "time": 177.69371061325074}, "bec0c6a95f": {"quality": 0.31897793263646923, "cost": 0.08918554800000002, "time": 219.01660284996032}, "bed888d4dc": {"quality": 0.6195121951219512, "cost": 0.012997296, "time": 392.78799629211426}, "bf45e407f6": {"quality": 0.7567479674796748, "cost": 0.085539992, "time": 158.9132725715637}, "bf5550f320": {"quality": 0.6613821138211382, "cost": 0.070068868, "time": 233.04684481620788}, "bf87e58322": {"quality": 0.6940650406504065, "cost": 0.06185992, "time": 188.9820168018341}, "bfed7670ed": {"quality": 0.645609756097561, "cost": 0.014454772, "time": 144.87740097045898}, "c0541e2220": {"quality": 0.6195121951219512, "cost": 0.06049530400000001, "time": 134.89153051376343}, "c0e10c0048": {"quality": 0.6195121951219512, "cost": 0.06394020800000001, "time": 253.89145169258117}, "c127509a7a": {"quality": 0.5921951219512195, "cost": 0.02507367, "time": 171.8149597644806}, "c13682c7c7": {"quality": 0.6195121951219512, "cost": 0.016085024, "time": 252.46734075546266}, "c13d6e78e9": {"quality": 0.4615098722415795, "cost": 0.026846708000000004, "time": 158.4395161151886}, "c14ff3144d": {"quality": 0.6380487804878049, "cost": 0.002752848, "time": 124.53351817131042}, "c1e42ac47b": {"quality": 0.5833333333333334, "cost": 0.07545873600000001, "time": 234.8793641090393}, "c2949aa902": {"quality": 0.6728455284552846, "cost": 0.00916251, "time": 212.7397180557251}, "c31e956b35": {"quality": 0.6621718931475028, "cost": 0.014615412, "time": 238.76983637809752}, "c36b525dde": {"quality": 0.781869918699187, "cost": 0.00692364, "time": 160.97053718566895}, "c38326e2bd": {"quality": 0.4835772357723577, "cost": 0.07338262, "time": 231.5795979499817}, "c3ec2cec59": {"quality": 0.6329268292682928, "cost": 0.017786080000000003, "time": 169.43426485061644}, "c44720575f": {"quality": 0.5554355400696864, "cost": 0.035053448, "time": 162.69704794883728}, "c48ecefab6": {"quality": 0.6195121951219512, "cost": 0.06189563200000001, "time": 231.7577772140503}, "c4a64eb40f": {"quality": 0.5757375145180024, "cost": 0.087534268, "time": 171.7744441986084}, "c4a80d19b3": {"quality": 0.667862950058072, "cost": 0.017699555999999998, "time": 262.5382396697998}, "c4c2826afd": {"quality": 0.6195121951219512, "cost": 0.014996676, "time": 243.1779758453369}, "c4c94a5527": {"quality": 0.6195121951219512, "cost": 0.009552324000000001, "time": 171.8789219379425}, "c4e75ee9ba": {"quality": 0.6092334494773519, "cost": 0.14632424400000002, "time": 163.68729362487792}, "c4f3e7665d": {"quality": 0.6195121951219512, "cost": 0.006700248000000001, "time": 171.2033597946167}, "c5471bef57": {"quality": 0.6195121951219512, "cost": 0.06709224, "time": 360.6734181404114}, "c54a408db7": {"quality": 0.4271544715447154, "cost": 0.061197178000000005, "time": 205.84643664360047}, "c59cc41335": {"quality": 0.6195121951219512, "cost": 0.060401755999999994, "time": 227.63298444747926}, "c5a0b065e0": {"quality": 0.6585365853658537, "cost": 0.020608076000000003, "time": 100.40361194610595}, "c5a16b834a": {"quality": 0.6756097560975609, "cost": 0.020331156, "time": 377.4331964969635}, "c5fbe2076f": {"quality": 0.2873170731707317, "cost": 0.024696848, "time": 235.4668863296509}, "c617370f6b": {"quality": 0.6195121951219512, "cost": 0.05902978000000001, "time": 201.3127824783325}, "c67f782c7f": {"quality": 0.6860975609756098, "cost": 0.064923248, "time": 186.35708870887757}, "c691a29c42": {"quality": 0.5146341463414634, "cost": 0.02263088, "time": 342.815619802475}, "c6a339987c": {"quality": 0.6195121951219512, "cost": 0.014977467999999999, "time": 256.19790320396424}, "c772ff3704": {"quality": 0.5390243902439025, "cost": 0.012446268, "time": 256.34659972190855}, "c7e3f348c2": {"quality": 0.7056910569105692, "cost": 0.07413946400000002, "time": 226.76761050224303}, "c823589ab6": {"quality": 0.6195121951219512, "cost": 0.060011588000000005, "time": 169.34995718002318}, "c82f834e85": {"quality": 0.6195121951219512, "cost": 0.013014976000000001, "time": 360.72568283081057}, "c85099881f": {"quality": 0.6214634146341463, "cost": 0.019014336, "time": 91.83121838569642}, "c935a33384": {"quality": 0.47804878048780486, "cost": 0.021866696, "time": 201.4007071018219}, "ca3177461f": {"quality": 0.6390243902439025, "cost": 0.064102784, "time": 260.361168384552}, "caa7c0bd6b": {"quality": 0.614308943089431, "cost": 0.013427039999999998, "time": 201.50019497871398}, "cac6b051e9": {"quality": 0.6777119628339141, "cost": 0.012200891999999998, "time": 235.3216923236847}, "cacb342f64": {"quality": 0.3350406504065041, "cost": 0.09074681200000001, "time": 199.62914476394653}, "cb9948679c": {"quality": 0.6195121951219512, "cost": 0.060963451999999994, "time": 209.66846399307252}, "cbb5eb0e74": {"quality": 0.36753774680603957, "cost": 0.012385512, "time": 269.31641936302185}, "cbc32cbeff": {"quality": 0.09656213704994195, "cost": 0.012335669999999998, "time": 223.05522747039794}, "cbd4461293": {"quality": 0.4478861788617886, "cost": 0.037316864, "time": 1704.694543504715}, "cbe2318045": {"quality": 0.35097560975609754, "cost": 0.011905024, "time": 143.2994128704071}, "cc886fe337": {"quality": 0.6195121951219512, "cost": 0.008570712000000001, "time": 189.0461087703705}, "cc9a6248a0": {"quality": 0.8031823461091754, "cost": 0.06325050000000002, "time": 65.81817102432251}, "ccb2335b3f": {"quality": 0.30666666666666675, "cost": 0.066565, "time": 270.24665660858153}, "ccdf03a55b": {"quality": 0.554239256678281, "cost": 0.066687724, "time": 194.36894397735597}, "ccf72745c1": {"quality": 0.5723577235772358, "cost": 0.015334892, "time": 180.34684844017028}, "cd1d418732": {"quality": 0.6195121951219512, "cost": 0.007930735999999999, "time": 90.1477038860321}, "cd23c79db1": {"quality": 0.6195121951219512, "cost": 0.013173464000000001, "time": 219.91192269325256}, "cd64fbfcd9": {"quality": 0.6552845528455286, "cost": 0.010721148, "time": 215.86649370193481}, "cd85a01e81": {"quality": 0.6195121951219512, "cost": 0.066680636, "time": 309.61991782188414}, "ce4bc5f348": {"quality": 0.4926829268292683, "cost": 0.004973436, "time": 122.72588214874267}, "ce980cf86f": {"quality": 0.3678281068524971, "cost": 0.006999414, "time": 186.7628839492798}, "ceae8b8bb9": {"quality": 0.569965156794425, "cost": 0.040919136, "time": 278.74190187454224}, "cecca90dd2": {"quality": 0.7816376306620209, "cost": 0.009244559999999999, "time": 204.2295413017273}, "cf9538faf0": {"quality": 0.5823577235772358, "cost": 0.013121356, "time": 168.41699748039247}, "cf9d2e224c": {"quality": 0.5317073170731708, "cost": 0.008401064, "time": 129.19066014289857}, "cfd36f3a8c": {"quality": 0.5922764227642275, "cost": 0.026429034, "time": 197.97676906585693}, "cffa29a6ef": {"quality": 0.6403252032520326, "cost": 0.0011394740000000001, "time": 84.72708497047424}, "d03596c3de": {"quality": 0.6584552845528455, "cost": 0.017459960000000004, "time": 243.87012338638306}, "d07b766487": {"quality": 0.5414634146341464, "cost": 0.06639410400000001, "time": 248.754083442688}, "d0a0a66d75": {"quality": 0.4197560975609756, "cost": 0.037255556, "time": 155.2362714290619}, "d0ce31134c": {"quality": 0.6195121951219512, "cost": 0.07035569600000001, "time": 244.35629963874817}, "d0f9633442": {"quality": 0.6195121951219512, "cost": 0.010673088, "time": 174.68734726905822}, "d216eab7d8": {"quality": 0.3921138211382114, "cost": 0.015362728, "time": 177.72364864349365}, "d266c19ac8": {"quality": 0.5590592334494773, "cost": 0.010259069999999999, "time": 249.80076293945314}, "d26a70179a": {"quality": 0.6736701509872242, "cost": 0.072242652, "time": 281.9195426940918}, "d2af24b59e": {"quality": 0.37926829268292683, "cost": 0.012534632, "time": 162.15444717407226}, "d2f2dd5cd4": {"quality": 0.6195121951219512, "cost": 0.033032736, "time": 161.3020932674408}, "d302278f85": {"quality": 0.6195121951219512, "cost": 0.065508068, "time": 281.0260838031769}, "d37dcaea30": {"quality": 0.5957026713124275, "cost": 0.042292012000000004, "time": 254.16097497940063}, "d3a2d50bd7": {"quality": 0.5393263646922184, "cost": 0.008568708, "time": 180.23848376274108}, "d3d4185487": {"quality": 0.6195121951219512, "cost": 0.063731828, "time": 186.93449816703796}, "d3db4cf84d": {"quality": 0.6195121951219512, "cost": 0.007814196, "time": 126.7787591457367}, "d402233b53": {"quality": 0.7804761904761903, "cost": 0.04559879200000001, "time": 268.7110698223114}, "d43fafa19e": {"quality": 0.6195121951219512, "cost": 0.006752928, "time": 137.8223207473755}, "d446a75eb7": {"quality": 0.508432055749129, "cost": 0.07295704800000001, "time": 211.27834362983702}, "d48ead13da": {"quality": 0.6195121951219512, "cost": 0.010855976, "time": 194.97514882087708}, "d5016f4538": {"quality": 0.6195121951219512, "cost": 0.006847740000000001, "time": 154.1272620677948}, "d55a3613b0": {"quality": 0.7033565621370499, "cost": 0.09393361200000001, "time": 280.48901586532594}, "d58036ba66": {"quality": 0.6341463414634146, "cost": 0.012329896000000002, "time": 182.65342464447022}, "d5a84c782e": {"quality": 0.7512311265969802, "cost": 0.06159429000000001, "time": 238.61238617897033}, "d5b2eef11c": {"quality": 0.6305458768873403, "cost": 0.059769270000000006, "time": 148.21854801177977}, "d6040140b9": {"quality": 0.6772357723577236, "cost": 0.025150488000000006, "time": 293.03985538482664}, "d65185c1a4": {"quality": 0.5714750290360046, "cost": 0.011127215999999999, "time": 211.66472873687744}, "d667351f33": {"quality": 0.6195121951219512, "cost": 0.07187473600000001, "time": 284.58656883239746}, "d6bd3b66ba": {"quality": 0.6195121951219512, "cost": 0.0062973000000000005, "time": 179.81442737579346}, "d6c4e48eeb": {"quality": 0.3821138211382114, "cost": 0.013097992, "time": 185.88441767692566}, "d6cbf265ee": {"quality": 0.553170731707317, "cost": 0.027157003999999995, "time": 2382.298054933548}, "d705447fd7": {"quality": 0.6195121951219512, "cost": 0.06927182000000001, "time": 274.04458661079406}, "d73a9aab4e": {"quality": 0.6195121951219512, "cost": 0.00601782, "time": 119.8721248626709}, "d752c30d07": {"quality": 0.6491869918699187, "cost": 0.029268951999999997, "time": 187.99695973396302}, "d782682359": {"quality": 0.48292682926829267, "cost": 0.070581532, "time": 240.62954745292663}, "d7c0972014": {"quality": 0.46747967479674796, "cost": 0.014235999999999999, "time": 146.19986548423768}, "d867525748": {"quality": 0.2914634146341463, "cost": 0.031817048, "time": 176.6470585823059}, "d87eb775da": {"quality": 0.5716260162601626, "cost": 0.011791225999999998, "time": 171.5166805744171}, "d8bab6c09b": {"quality": 0.5560975609756098, "cost": 0.0201636, "time": 262.4358974933624}, "d8bcac36e8": {"quality": 0.6195121951219512, "cost": 0.014129348, "time": 178.82634310722352}, "d8eadc0190": {"quality": 0.6644715447154471, "cost": 0.08632676000000002, "time": 188.2232988357544}, "d96677d8d4": {"quality": 0.7195005807200929, "cost": 0.010380108, "time": 202.3421525001526}, "d98f22270e": {"quality": 0.46382113821138216, "cost": 0.06577788, "time": 195.36387667655944}, "d9e2bb21a3": {"quality": 0.5120789779326365, "cost": 0.011266182, "time": 204.83109889030456}, "da95deeb20": {"quality": 0.3967479674796748, "cost": 0.05949913, "time": 146.4172206401825}, "daaadadcc9": {"quality": 0.6085365853658538, "cost": 0.014349220000000001, "time": 236.59604306221007}, "daf855e065": {"quality": 0.7283739837398373, "cost": 0.01084773, "time": 254.38643417358398}, "db00594832": {"quality": 0.7669570267131242, "cost": 0.082393492, "time": 131.31271381378173}, "db19e677c4": {"quality": 0.816341463414634, "cost": 0.08040591200000001, "time": 186.9021366119385}, "db3c035639": {"quality": 0.6329268292682927, "cost": 0.07513583600000001, "time": 251.00368614196776}, "db41487005": {"quality": 0.6195121951219512, "cost": 0.05802746000000001, "time": 196.43507223129274}, "db6a7482fd": {"quality": 0.6803716608594658, "cost": 0.056941420000000006, "time": 138.47659482955933}, "db9060cd27": {"quality": 0.42195121951219516, "cost": 0.027881287999999997, "time": 1896.3412520885468}, "dbce95a072": {"quality": 0.5060975609756098, "cost": 0.012663032000000001, "time": 179.2968252182007}, "dc195abe5e": {"quality": 0.31232288037166084, "cost": 0.062047202, "time": 256.8396565437317}, "dc3f4b7138": {"quality": 0.6195121951219512, "cost": 0.05797826800000001, "time": 188.94316940307618}, "dc66bccb1c": {"quality": 0.3678048780487805, "cost": 0.01096708, "time": 168.7502061367035}, "dc90065dea": {"quality": 0.36016260162601627, "cost": 0.014289246, "time": 199.27637152671815}, "dd0d70fedd": {"quality": 0.7897677119628339, "cost": 0.005333369999999999, "time": 160.19438481330872}, "dddc76b3ca": {"quality": 0.32487804878048776, "cost": 0.06413708, "time": 203.57222208976745}, "de18bf45e1": {"quality": 0.5365040650406504, "cost": 0.009448855999999999, "time": 966.3698208808898}, "de1e56370f": {"quality": 0.6371544715447155, "cost": 0.007415147999999999, "time": 141.48761720657347}, "df2160ecc8": {"quality": 0.4975609756097561, "cost": 0.06514112000000001, "time": 184.16321535110472}, "dfda94bd2a": {"quality": 0.6195121951219512, "cost": 0.0029004839999999996, "time": 111.35115175247192}, "dff452a9ca": {"quality": 0.6195121951219512, "cost": 0.00818748, "time": 128.0037736415863}, "e09f75d9d6": {"quality": 0.5954123112659697, "cost": 0.07994701400000001, "time": 174.90588331222534}, "e0b6a99753": {"quality": 0.6808943089430894, "cost": 0.022907588, "time": 75.86744494438172}, "e0cf6587a7": {"quality": 0.38739837398373983, "cost": 0.06783974000000001, "time": 224.5930316925049}, "e0de4a5929": {"quality": 0.28844367015098726, "cost": 0.070857196, "time": 237.39370694160462}, "e1356fb426": {"quality": 0.563054587688734, "cost": 0.06065425, "time": 185.1619038105011}, "e20ba014a1": {"quality": 0.6341463414634146, "cost": 0.022406492, "time": 231.54046459197997}, "e21806e3bc": {"quality": 0.6686991869918699, "cost": 0.030775240000000002, "time": 153.30845246315002}, "e24e97564c": {"quality": 0.5788269454123112, "cost": 0.065976868, "time": 234.19773464202882}, "e2673c1ec8": {"quality": 0.5588617886178862, "cost": 0.07093568800000001, "time": 268.0965113162994}, "e26c7bfbdb": {"quality": 0.526829268292683, "cost": 0.010329528, "time": 162.32453393936157}, "e2f9980b06": {"quality": 0.8120905923344948, "cost": 0.041606056, "time": 251.1425283432007}, "e3445f7632": {"quality": 0.7865737514518002, "cost": 0.02273798, "time": 124.58966102600098}, "e35e5f81a7": {"quality": 0.6195121951219512, "cost": 0.012546798000000001, "time": 182.4151572227478}, "e376ac53e7": {"quality": 0.5323577235772358, "cost": 0.031615856, "time": 169.9604299068451}, "e3d8bb56da": {"quality": 0.7227758420441346, "cost": 0.03464479, "time": 173.92777223587035}, "e3df4cf041": {"quality": 0.6195121951219512, "cost": 0.013172260000000002, "time": 228.6481415748596}, "e47dc3abca": {"quality": 0.5421602787456445, "cost": 0.01662284, "time": 196.10405125617982}, "e4b9d4fb41": {"quality": 0.6195121951219512, "cost": 0.013018407999999999, "time": 231.83416152000427}, "e510bda989": {"quality": 0.4415447154471545, "cost": 0.12802820799999998, "time": 1276.7408336162566}, "e517cd2222": {"quality": 0.3045528455284553, "cost": 0.020794063999999998, "time": 1299.5594879627229}, "e51b01f418": {"quality": 0.6195121951219512, "cost": 0.011492568, "time": 202.7894808292389}, "e520dfae5b": {"quality": 0.6439024390243903, "cost": 0.011328468, "time": 263.53743448257444}, "e521c9b7e4": {"quality": 0.7932520325203252, "cost": 0.06690654, "time": 156.73078799247742}, "e54097ad5d": {"quality": 0.5923344947735192, "cost": 0.062158128, "time": 209.91472067832947}, "e56a16ca66": {"quality": 0.4848315911730546, "cost": 0.004748472, "time": 137.46008925437928}, "e5c4abf7ce": {"quality": 0.35598141695702673, "cost": 0.06409228000000002, "time": 256.3565216064453}, "e5d4689312": {"quality": 0.6195121951219512, "cost": 0.06653434, "time": 235.4001616001129}, "e62a7b27ae": {"quality": 0.6195121951219512, "cost": 0.035626552, "time": 195.45834450721742}, "e6a7aff3bc": {"quality": 0.6195121951219512, "cost": 0.06874828800000002, "time": 295.3631275177002}, "e736999157": {"quality": 0.6195121951219512, "cost": 0.06437495600000001, "time": 288.5091665744782}, "e7517a8ce0": {"quality": 0.8125435540069686, "cost": 0.08348902, "time": 200.60669808387757}, "e7520ca5ac": {"quality": 0.36312427409988385, "cost": 0.06479258000000002, "time": 261.0504195690155}, "e7e94ab7a5": {"quality": 0.591869918699187, "cost": 0.019925159999999997, "time": 2488.357094335556}, "e887ddf5cc": {"quality": 0.48739837398373986, "cost": 0.09377338400000002, "time": 253.3760078907013}, "e94fb5a295": {"quality": 0.6195121951219512, "cost": 0.07366761200000001, "time": 223.6755922794342}, "ea6ecc5653": {"quality": 0.6166666666666667, "cost": 0.010067135999999999, "time": 190.92216954231262}, "ea8bcb3ae2": {"quality": 0.45772357723577234, "cost": 0.022977280000000003, "time": 212.54974527359008}, "ebbe8b6c4f": {"quality": 0.6195121951219512, "cost": 0.009877384, "time": 160.62470216751097}, "ebdf3abff2": {"quality": 0.686260162601626, "cost": 0.019338396, "time": 179.08990707397462}, "ec55dba809": {"quality": 0.4276422764227642, "cost": 0.068014448, "time": 230.82128653526306}, "ecb5f78f37": {"quality": 0.6195121951219512, "cost": 0.017395611999999998, "time": 249.2554892539978}, "ecda3d74cb": {"quality": 0.6195121951219512, "cost": 0.061945256000000004, "time": 200.0551459789276}, "ece10c0388": {"quality": 0.7655168408826946, "cost": 0.129639792, "time": 172.15098700523376}, "ed6b5480a5": {"quality": 0.47195121951219515, "cost": 0.006612492, "time": 117.65485558509826}, "eda630dc85": {"quality": 0.6290127758420442, "cost": 0.013249908000000001, "time": 202.01231064796448}, "edaaee5ed4": {"quality": 0.5414634146341464, "cost": 0.011020607999999998, "time": 124.82250752449036}, "edb2b764aa": {"quality": 0.6329268292682928, "cost": 0.018810900000000002, "time": 160.87249393463134}, "edc52339db": {"quality": 0.6195121951219512, "cost": 0.0179609, "time": 246.9092752456665}, "ede7071775": {"quality": 0.7035423925667827, "cost": 0.13207611000000002, "time": 200.41918315887452}, "ee46042c5d": {"quality": 0.6195121951219512, "cost": 0.016286580000000002, "time": 244.87669949531556}, "ee68c51f73": {"quality": 0.6029500580720093, "cost": 0.07541250000000001, "time": 109.02409811019898}, "ee7b726747": {"quality": 0.7833797909407665, "cost": 0.039892712, "time": 251.92134594917297}, "eec5f32da9": {"quality": 0.81595818815331, "cost": 0.0879383, "time": 286.01010518074037}, "eed40e4378": {"quality": 0.6613821138211382, "cost": 0.06929234, "time": 282.0774456501007}, "eef12d478b": {"quality": 0.46260162601626015, "cost": 0.009616956, "time": 204.5144693851471}, "ef37b3e0be": {"quality": 0.6195121951219512, "cost": 0.012494243999999998, "time": 257.47455110549924}, "ef43d497f1": {"quality": 0.40569105691056906, "cost": 0.016996992, "time": 246.4581404685974}, "ef4d4c4a62": {"quality": 0.4484552845528455, "cost": 0.015539336, "time": 182.2000419616699}, "ef9a651425": {"quality": 0.7040650406504065, "cost": 0.07579404, "time": 250.92651562690736}, "f0655621af": {"quality": 0.4265853658536585, "cost": 0.15761675999999997, "time": 2013.6334864139558}, "f076b4c9ae": {"quality": 0.7834262485481996, "cost": 0.015863708, "time": 187.7628330230713}, "f11eddb4ed": {"quality": 0.20284552845528453, "cost": 0.016428986, "time": 237.75844135284424}, "f1408da253": {"quality": 0.6967479674796748, "cost": 0.014442047999999999, "time": 249.8467248916626}, "f18cf41929": {"quality": 0.6195121951219512, "cost": 0.003961836, "time": 117.34907512664795}, "f1aa0b0b42": {"quality": 0.5720325203252032, "cost": 0.043219422, "time": 146.36151003837585}, "f1bda127f6": {"quality": 0.46178861788617886, "cost": 0.017929852000000003, "time": 197.9198618412018}, "f1f373e58e": {"quality": 0.7831010452961673, "cost": 0.09002725200000002, "time": 185.47337765693663}, "f2a2e91541": {"quality": 0.4859117305458769, "cost": 0.012855611999999999, "time": 232.05357608795165}, "f2c04ed1c8": {"quality": 0.405609756097561, "cost": 0.0067632479999999995, "time": 126.54541292190552}, "f2cf5db12d": {"quality": 0.19959349593495934, "cost": 0.013177552, "time": 168.4030219078064}, "f366c0dd10": {"quality": 0.6195121951219512, "cost": 0.06272198, "time": 196.8466109275818}, "f4303a5b4f": {"quality": 0.6103252032520325, "cost": 0.09301452800000001, "time": 231.81033272743224}, "f437481e3b": {"quality": 0.5909756097560975, "cost": 0.019717612, "time": 227.29155945777893}, "f4bc6b63a7": {"quality": 0.571788617886179, "cost": 0.12058799000000002, "time": 197.6338578224182}, "f4dc556633": {"quality": 0.6195121951219512, "cost": 0.06797158, "time": 289.321187877655}, "f4deb72db6": {"quality": 0.6195121951219512, "cost": 0.0105891, "time": 200.89887857437134}, "f4ef2b9c33": {"quality": 0.5439837398373983, "cost": 0.03817632000000001, "time": 213.8931649684906}, "f566d6d6a1": {"quality": 0.4036585365853659, "cost": 0.062300763999999995, "time": 201.07639598846436}, "f5b9a94dcc": {"quality": 0.6146341463414634, "cost": 0.04594065199999999, "time": 1847.3314466953277}, "f5c27e7172": {"quality": 0.6195121951219512, "cost": 0.007075463999999999, "time": 232.92631516456603}, "f5e53d963b": {"quality": 0.6195121951219512, "cost": 0.004926876000000001, "time": 146.53346581459044}, "f614235c15": {"quality": 0.546829268292683, "cost": 0.019139696, "time": 1690.8127262592316}, "f6546149e3": {"quality": 0.7115214866434378, "cost": 0.08226788, "time": 214.23944363594055}, "f74ec023e4": {"quality": 0.6195121951219512, "cost": 0.016337988, "time": 300.67309379577637}, "f7b048bd54": {"quality": 0.46829268292682935, "cost": 0.016687760000000003, "time": 214.77183957099913}, "f7c4df993e": {"quality": 0.6378513356562137, "cost": 0.01224465, "time": 182.30367503166198}, "f854533145": {"quality": 0.6585365853658537, "cost": 0.017361696, "time": 270.88364191055297}, "f89b8a1930": {"quality": 0.8136236933797909, "cost": 0.040442136000000004, "time": 276.7925349235535}, "f93d9a2693": {"quality": 0.37878048780487805, "cost": 0.008193323999999998, "time": 204.3297384738922}, "f97d91a249": {"quality": 0.3878048780487805, "cost": 0.06316157600000001, "time": 184.88821873664855}, "f99096d89c": {"quality": 0.6195121951219512, "cost": 0.014407992, "time": 267.10197510719297}, "f9e8e221f3": {"quality": 0.39916376306620205, "cost": 0.010630103999999998, "time": 213.8131926059723}, "fa38879eab": {"quality": 0.6504065040650406, "cost": 0.021828032000000004, "time": 295.17917666435244}, "fa71111570": {"quality": 0.5853658536585366, "cost": 0.069471068, "time": 280.6139543056488}, "fa7882d46b": {"quality": 0.34783972125435536, "cost": 0.016001058000000002, "time": 247.1370331764221}, "fa906520d1": {"quality": 0.6655981416957026, "cost": 0.018073107999999997, "time": 259.80645098686216}, "faabebaa30": {"quality": 0.22723577235772358, "cost": 0.015047418000000002, "time": 238.21279168128967}, "fb0339a7d0": {"quality": 0.7656794425087108, "cost": 0.021738743999999997, "time": 273.6737798213959}, "fb216ad6b3": {"quality": 0.3592682926829268, "cost": 0.05636392800000001, "time": 978.5727415084839}, "fb6216880a": {"quality": 0.6195121951219512, "cost": 0.016447008, "time": 279.96212286949157}, "fba499b89d": {"quality": 0.6195121951219512, "cost": 0.06753407200000001, "time": 276.17343015670775}, "fbc010a368": {"quality": 0.6195121951219512, "cost": 0.06443590400000002, "time": 217.46284394264222}, "fbc02e2e07": {"quality": 0.6195121951219512, "cost": 0.061001164, "time": 259.36248960494993}, "fbd6c45271": {"quality": 0.24020905923344946, "cost": 0.015250424000000002, "time": 285.5677849769592}, "fc0a156e16": {"quality": 0.7864692218350754, "cost": 0.034151164, "time": 244.54232273101806}, "fc1fd5bf54": {"quality": 0.6195121951219512, "cost": 0.010393596, "time": 302.0887975215912}, "fc6967a75b": {"quality": 0.6195121951219512, "cost": 0.06284471200000001, "time": 293.1112622261047}, "fc73c3b0fa": {"quality": 0.6592682926829267, "cost": 0.025536140000000002, "time": 98.14458861351014}, "fce38334b2": {"quality": 0.6195121951219512, "cost": 0.01724038, "time": 267.3305795669556}, "fce5fca128": {"quality": 0.4260278745644599, "cost": 0.066997868, "time": 262.8612917900085}, "fd0709359e": {"quality": 0.6446341463414634, "cost": 0.001408038, "time": 79.83381242752075}, "fd1f809d64": {"quality": 0.8097560975609757, "cost": 0.07385033799999999, "time": 213.83500204086303}, "fd2c994a9d": {"quality": 0.6390243902439025, "cost": 0.06503392399999999, "time": 284.80486459732055}, "fddccfbf94": {"quality": 0.46829268292682935, "cost": 0.014656296000000003, "time": 217.74681878089905}, "fe7fa741b4": {"quality": 0.48540069686411147, "cost": 0.034931979999999994, "time": 209.350274848938}, "fe9e1fec71": {"quality": 0.5036585365853659, "cost": 0.067148512, "time": 212.58801732063293}, "fea4734c09": {"quality": 0.4943902439024391, "cost": 0.008797415999999999, "time": 129.66075100898743}, "fef1ca27fa": {"quality": 0.4926829268292683, "cost": 0.020025176, "time": 248.60985913276673}, "ff11cb6a7a": {"quality": 0.39739837398373984, "cost": 0.0197844, "time": 228.27161712646483}, "ff171e34e2": {"quality": 0.6195121951219512, "cost": 0.007519518, "time": 156.71968116760254}, "ff1c958e21": {"quality": 0.6341463414634146, "cost": 0.0048062339999999995, "time": 124.33939328193665}, "ff8df4ace9": {"quality": 0.5845528455284553, "cost": 0.0622996, "time": 187.53624334335328}, "ff8e68049a": {"quality": 0.28821138211382114, "cost": 0.010620372, "time": 159.36370429992675}} ================================================ FILE: abacus-research/cuad_data_loader.py ================================================ """ Shared CUAD data loading utilities to replace HuggingFace datasets. All CUAD scripts should import from this module. """ import json import os import numpy as np # Default data directory DEFAULT_DATA_DIR = "cuad-data" def load_cuad_data(split="test", data_dir=None): """ Load CUAD dataset from local JSON files. Args: split: "train" or "test" data_dir: Directory containing CUAD JSON files (default: "cuad-data") Returns: List of dictionaries with CUAD data in flat format """ if data_dir is None: data_dir = DEFAULT_DATA_DIR if split == "train": file_path = os.path.join(data_dir, "train_separate_questions.json") else: file_path = os.path.join(data_dir, "test.json") if not os.path.exists(file_path): raise FileNotFoundError( f"CUAD data file not found at {file_path}. " f"Please run 'python setup_cuad_data.py' first to download the data." ) with open(file_path) as f: raw_data = json.load(f) # Convert to flat format dataset = [] for article in raw_data["data"]: title = article.get("title", "").strip() for paragraph in article["paragraphs"]: context = paragraph["context"].strip() for qa in paragraph["qas"]: dataset.append({ "id": qa["id"], "title": title, "context": context, "question": qa["question"].strip(), "answers": qa.get("answers", []) }) return dataset def get_unique_contracts(dataset): """Get list of unique contract titles from dataset.""" contract_titles = [] for row in dataset: if row["title"] not in contract_titles: contract_titles.append(row["title"]) return contract_titles def filter_by_contracts(dataset, contract_titles): """Filter dataset to only include specified contracts.""" return [row for row in dataset if row["title"] in contract_titles] def sample_contracts(dataset, num_contracts, seed=42): """ Sample a subset of contracts from the dataset. Args: dataset: CUAD dataset num_contracts: Number of contracts to sample seed: Random seed for reproducibility Returns: Filtered dataset with only the sampled contracts """ contract_titles = get_unique_contracts(dataset) # Shuffle and sample rng = np.random.default_rng(seed=seed) rng.shuffle(contract_titles) sampled_titles = contract_titles[:num_contracts] return filter_by_contracts(dataset, sampled_titles), sampled_titles ================================================ FILE: abacus-research/download_embeddings_and_mmqa.sh ================================================ #!/bin/bash wget -nc https://palimpzest-workloads.s3.us-east-1.amazonaws.com/abacus-data.tar.gz tar -xzf abacus-data.tar.gz ================================================ FILE: abacus-research/helper-scripts/biodex-gen-index.py ================================================ import os import time import chromadb import chromadb.utils.embedding_functions as embedding_functions import numpy as np from openai import OpenAI from tqdm import tqdm # NOTE: this script is meant to be run from the root of the repository if __name__ == "__main__": # initialize openai client openai_client = OpenAI() # load reaction terms reaction_terms = [] with open("testdata/reaction_terms.txt") as f: for line in f: reaction_terms.append(line.strip()) # create directory for embeddings os.makedirs("testdata/reaction-term-embeddings/", exist_ok=True) # generate embeddings in batches of 1000 at a time indices = np.linspace(0, len(reaction_terms), len(reaction_terms)//1000, dtype=int) total_embeds = len(indices) print(f"Generating {total_embeds} embeddings...") gen_indices = [] for iter_idx, start_idx in tqdm(enumerate(indices), total=total_embeds): # check if embedding needs to be computed end_idx = indices[iter_idx + 1] if iter_idx + 1 < len(indices) else None filename = f"testdata/reaction-term-embeddings/{start_idx}_{end_idx}.npy" if end_idx is not None and not os.path.exists(filename): # generate embeddings batch = reaction_terms[start_idx:end_idx] resp = openai_client.embeddings.create(input=batch, model="text-embedding-3-small") embeddings = [item.embedding for item in resp.data] # save embeddings to disk with open(filename, "wb") as f: np.save(f, np.array(embeddings)) gen_indices.append((start_idx, end_idx)) time.sleep(1) print("Done generating embeddings.") # initialize chroma client chroma_client = chromadb.PersistentClient(".chroma-biodex") # initialize embedding function openai_ef = embedding_functions.OpenAIEmbeddingFunction( api_key=os.environ["OPENAI_API_KEY"], model_name="text-embedding-3-small" ) # create a collection collection = chroma_client.get_or_create_collection( name="biodex-reaction-terms", embedding_function=openai_ef, metadata={"hnsw:space": "cosine"}, ) # insert documents in batches total_inserts = len(gen_indices) print(f"Inserting {total_inserts} batches into the collection...") for start_idx, end_idx in tqdm(gen_indices, total=total_inserts): embeddings = np.load(f"testdata/reaction-term-embeddings/{start_idx}_{end_idx}.npy") collection.add( documents=reaction_terms[start_idx:end_idx], embeddings=embeddings.tolist(), ids=[f"id{idx}" for idx in range(start_idx, end_idx)] ) ================================================ FILE: abacus-research/helper-scripts/generate-prior-stats-biodex-first-convert.py ================================================ """ NOTE: this script worked with the tag `abacus-paper-experiments` but is no longer compatible with the main branch. """ import argparse import json import os import time import datasets # from ragatouille import RAGPretrainedModel import palimpzest as pz from palimpzest.constants import Model biodex_entry_cols = [ {"name": "pmid", "type": str, "desc": "The PubMed ID of the medical paper"}, {"name": "title", "type": str, "desc": "The title of the medical paper"}, {"name": "abstract", "type": str, "desc": "The abstract of the medical paper"}, {"name": "fulltext", "type": str, "desc": "The full text of the medical paper, which contains information relevant for creating a drug safety report."}, ] biodex_reactions_cols = [ {"name": "reactions", "type": list[str], "desc": "The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible."}, ] class BiodexDataset(pz.IterDataset): def __init__( self, rp_at_k: int = 5, num_samples: int = 5, split: str = "test", shuffle: bool = False, seed: int = 42, ): super().__init__(id=f"biodex-{split}", schema=biodex_entry_cols) self.dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split=split).to_pandas() if shuffle: self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient="records") else: self.dataset = self.dataset.to_dict(orient="records")[:num_samples] self.rp_at_k = rp_at_k self.num_samples = num_samples self.shuffle = shuffle self.seed = seed self.split = split def compute_label(self, entry: dict) -> dict: """Compute the label for a BioDEX report given its entry in the dataset.""" reactions_lst = [ reaction.strip().lower().replace("'", "").replace("^", "") for reaction in entry["reactions"].split(",") ] label_dict = {"reactions": reactions_lst} return label_dict @staticmethod def term_recall(preds: list | None, targets: list): if preds is None: return 0.0 try: # normalize terms in each list pred_terms = set([ term.strip() for pred in preds for term in pred.lower().replace("'", "").replace("^", "").split(" ") ]) target_terms = ([ term.strip() for target in targets for term in target.lower().replace("'", "").replace("^", "").split(" ") ]) # compute term recall and return intersect = pred_terms.intersection(target_terms) term_recall = len(intersect) / len(target_terms) return term_recall except Exception: os.makedirs("term-recall-eval-errors", exist_ok=True) ts = time.time() with open(f"term-recall-eval-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def __len__(self): return len(self.dataset) def __getitem__(self, idx: int): # get entry entry = self.dataset[idx] # get input fields pmid = entry["pmid"] title = entry["title"] abstract = entry["abstract"] fulltext = entry["fulltext"] # create item with fields item = {"fields": {}, "labels": {}, "score_fn": {}} item["fields"]["pmid"] = pmid item["fields"]["title"] = title item["fields"]["abstract"] = abstract item["fields"]["fulltext"] = fulltext if self.split == "train": # add label info item["labels"] = self.compute_label(entry) # add scoring functions for list fields item["score_fn"]["reactions"] = BiodexDataset.term_recall return item if __name__ == "__main__": # parse arguments parser = argparse.ArgumentParser(description="Run a simple demo") parser.add_argument("--verbose", default=False, action="store_true", help="Print verbose output") parser.add_argument("--progress", default=True, action="store_true", help="Print progress output") args = parser.parse_args() # create directory for profiling data os.makedirs("priors-data", exist_ok=True) verbose = args.verbose progress = args.progress seed = 123 # NOTE: unique to cascades run execution_strategy = "parallel" sentinel_execution_strategy = "all" optimizer_strategy = "pareto" exp_name = f"biodex-priors-{optimizer_strategy}-seed{seed}-cascades" # NOTE: unique to cascades run if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") # create data source dataset = BiodexDataset( split="test", num_samples=1, shuffle=True, seed=seed, ) # create validation data source train_dataset = BiodexDataset( split="train", num_samples=5, shuffle=True, seed=seed, ) # construct plan plan = dataset plan = plan.sem_add_columns(biodex_reactions_cols) # only use final op quality use_final_op_quality = True # execute pz plan config = pz.QueryProcessorConfig( optimizer_strategy=optimizer_strategy, sentinel_execution_strategy=sentinel_execution_strategy, execution_strategy=execution_strategy, use_final_op_quality=use_final_op_quality, max_workers=64, verbose=verbose, available_models=[ # NOTE: unique to cascades run # Model.GPT_4o, Model.GPT_4o_MINI, Model.LLAMA3_2_3B, Model.LLAMA3_1_8B, Model.LLAMA3_3_70B, Model.LLAMA3_2_90B_V, # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments` # Model.DEEPSEEK_V3, Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B, ], allow_bonded_query=True, allow_critic=True, allow_mixtures=True, allow_rag_reduction=True, progress=progress, k=-1, j=-1, sample_budget=5*1014, seed=seed, exp_name=exp_name, ) data_record_collection = plan.run(config=config, train_dataset=train_dataset, validator=pz.Validator()) print(data_record_collection.to_df()) data_record_collection.to_df().to_csv(f"priors-data/{exp_name}-output.csv", index=False) # create filepaths for records and stats records_path = f"priors-data/{exp_name}-records.json" stats_path = f"priors-data/{exp_name}-profiling.json" # save record outputs record_jsons = [] for record in data_record_collection: record_dict = record.to_dict() record_dict = { k: v for k, v in record_dict.items() if k in ["pmid", "reactions"] } record_jsons.append(record_dict) with open(records_path, "w") as f: json.dump(record_jsons, f) # save statistics execution_stats_dict = data_record_collection.execution_stats.to_json() with open(stats_path, "w") as f: json.dump(execution_stats_dict, f) ================================================ FILE: abacus-research/helper-scripts/generate-prior-stats-biodex.py ================================================ """ NOTE: this script worked with the tag `abacus-paper-experiments` but is no longer compatible with the main branch. """ import argparse import json import os import time from functools import partial import chromadb import datasets from chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction # from ragatouille import RAGPretrainedModel import palimpzest as pz from palimpzest.constants import Model biodex_entry_cols = [ {"name": "pmid", "type": str, "desc": "The PubMed ID of the medical paper"}, {"name": "title", "type": str, "desc": "The title of the medical paper"}, {"name": "abstract", "type": str, "desc": "The abstract of the medical paper"}, {"name": "fulltext", "type": str, "desc": "The full text of the medical paper, which contains information relevant for creating a drug safety report."}, {"name": "reactions", "type": list[str], "desc": "The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible."}, ] biodex_reaction_labels_cols = [ {"name": "reaction_labels", "type": list[str], "desc": "Official terms for medical conditions listed in `reactions`"}, ] biodex_ranked_reactions_labels_cols = [ {"name": "ranked_reaction_labels", "type": list[str], "desc": "The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs."}, ] class BiodexDataset(pz.IterDataset): def __init__( self, rp_at_k: int = 5, num_samples: int = 5, split: str = "test", ): super().__init__(id=f"biodex-{split}", schema=biodex_entry_cols) if split == "test": self.dataset = datasets.load_dataset("BioDEX/BioDEX-Reactions", split=split).to_pandas().to_dict(orient="records")[:num_samples] else: with open('priors-data/source-idx-to-record-state-cascades.json') as f: # NOTE: unique to cascades run self.source_idx_to_record_state = json.load(f) self.dataset = [ self.source_idx_to_record_state[str(idx)] for idx in range(5) ] self.rp_at_k = rp_at_k self.num_samples = num_samples self.split = split def compute_label(self, entry: dict) -> dict: """Compute the label for a BioDEX report given its entry in the dataset.""" reactions_lst = [ reaction.strip().lower().replace("'", "").replace("^", "") for reaction in json.dumps(entry["reactions"]).split(",") ] label_dict = { "reaction_labels": reactions_lst, "ranked_reaction_labels": reactions_lst, } return label_dict @staticmethod def rank_precision_at_k(preds: list | None, targets: list, k: int): if preds is None: return 0.0 try: # lower-case each list preds = [pred.strip().lower().replace("'", "").replace("^", "") for pred in preds] targets = set([target.strip().lower().replace("'", "").replace("^", "") for target in targets]) # compute rank-precision at k rn = len(targets) denom = min(k, rn) total = 0.0 for i in range(k): total += preds[i] in targets if i < len(preds) else 0.0 return total / denom except Exception: os.makedirs("rp@k-errors", exist_ok=True) ts = time.time() with open(f"rp@k-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 @staticmethod def term_recall(preds: list | None, targets: list): if preds is None: return 0.0 try: # normalize terms in each list pred_terms = set([ term.strip() for pred in preds for term in pred.lower().replace("'", "").replace("^", "").split(" ") ]) target_terms = ([ term.strip() for target in targets for term in target.lower().replace("'", "").replace("^", "").split(" ") ]) # compute term recall and return intersect = pred_terms.intersection(target_terms) term_recall = len(intersect) / len(target_terms) return term_recall except Exception: os.makedirs("term-recall-eval-errors", exist_ok=True) ts = time.time() with open(f"term-recall-eval-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def __len__(self): return len(self.dataset) def __getitem__(self, idx: int): # get entry entry = self.dataset[idx] # get input fields pmid = entry["pmid"] title = entry["title"] abstract = entry["abstract"] fulltext = entry["fulltext"] reactions = entry["reactions"] # create item with fields item = {"fields": {}, "labels": {}, "score_fn": {}} item["fields"]["pmid"] = pmid item["fields"]["title"] = title item["fields"]["abstract"] = abstract item["fields"]["fulltext"] = fulltext item["fields"]["reactions"] = json.dumps(reactions) if self.split == "train": # add label info item["labels"] = self.compute_label(entry) # add scoring functions for list fields rank_precision_at_k = partial(BiodexDataset.rank_precision_at_k, k=self.rp_at_k) item["score_fn"]["reaction_labels"] = BiodexDataset.term_recall item["score_fn"]["ranked_reaction_labels"] = rank_precision_at_k return item if __name__ == "__main__": # parse arguments parser = argparse.ArgumentParser(description="Run a simple demo") parser.add_argument("--verbose", default=False, action="store_true", help="Print verbose output") parser.add_argument("--progress", default=True, action="store_true", help="Print progress output") args = parser.parse_args() # create directory for profiling data os.makedirs("priors-data", exist_ok=True) verbose = args.verbose progress = args.progress seed = 123 # NOTE: unique to cascades run execution_strategy = "parallel" sentinel_execution_strategy = "all" optimizer_strategy = "pareto" exp_name = f"biodex-priors-{optimizer_strategy}-seed{seed}-second-convert-cascades" # NOTE: unique to cascades run if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") # create data source dataset = BiodexDataset( split="test", num_samples=1, ) # create validation data source train_dataset = BiodexDataset( split="train", num_samples=5, ) # load index [text-embedding-3-small] chroma_client = chromadb.PersistentClient(".chroma-biodex") openai_ef = OpenAIEmbeddingFunction( api_key=os.environ["OPENAI_API_KEY"], model_name="text-embedding-3-small", ) index = chroma_client.get_collection("biodex-reaction-terms", embedding_function=openai_ef) def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]: # execute query with embeddings results = index.query(query, n_results=5) # get list of result terms with their cosine similarity scores final_results = [] for query_docs, query_distances in zip(results["documents"], results["distances"]): for doc, dist in zip(query_docs, query_distances): cosine_similarity = 1 - dist final_results.append({"content": doc, "similarity": cosine_similarity}) # sort the results by similarity score sorted_results = sorted(final_results, key=lambda result: result["similarity"], reverse=True) # remove duplicates sorted_results_set = set() final_sorted_results = [] for result in sorted_results: if result["content"] not in sorted_results_set: sorted_results_set.add(result["content"]) final_sorted_results.append(result["content"]) # return the top-k similar results and generation stats return {"reaction_labels": final_sorted_results[:k]} # construct plan plan = dataset.sem_topk( index=index, search_func=search_func, search_attr="reactions", output_attrs=biodex_reaction_labels_cols, ) plan = plan.sem_add_columns(biodex_ranked_reactions_labels_cols, depends_on=["title", "abstract", "fulltext", "reaction_labels"]) # only use final op quality use_final_op_quality = True # execute pz plan config = pz.QueryProcessorConfig( optimizer_strategy=optimizer_strategy, sentinel_execution_strategy=sentinel_execution_strategy, execution_strategy=execution_strategy, use_final_op_quality=use_final_op_quality, max_workers=64, verbose=verbose, available_models=[ # NOTE: unique to cascades run # Model.GPT_4o, Model.GPT_4o_MINI, Model.LLAMA3_2_3B, Model.LLAMA3_1_8B, Model.LLAMA3_3_70B, Model.LLAMA3_2_90B_V, # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments` # Model.DEEPSEEK_V3, Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B, ], allow_bonded_query=True, allow_critic=True, allow_mixtures=True, allow_rag_reduction=True, progress=progress, k=-1, j=-1, sample_budget=5*1014 + 5*7, seed=seed, exp_name=exp_name, ) data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=pz.Validator()) print(data_record_collection.to_df()) data_record_collection.to_df().to_csv(f"priors-data/{exp_name}-output.csv", index=False) # create filepaths for records and stats records_path = f"priors-data/{exp_name}-records.json" stats_path = f"priors-data/{exp_name}-profiling.json" # save record outputs record_jsons = [] for record in data_record_collection: record_dict = record.to_dict() record_dict = { k: v for k, v in record_dict.items() if k in ["pmid", "reactions"] } record_jsons.append(record_dict) with open(records_path, "w") as f: json.dump(record_jsons, f) # save statistics execution_stats_dict = data_record_collection.execution_stats.to_json() with open(stats_path, "w") as f: json.dump(execution_stats_dict, f) ================================================ FILE: abacus-research/helper-scripts/generate-prior-stats-cuad.py ================================================ """ NOTE: this script worked with the tag `abacus-paper-experiments` but is no longer compatible with the main branch. """ import argparse import json import os import string from functools import partial import datasets import numpy as np import pandas as pd import palimpzest as pz from palimpzest.constants import Model cuad_categories = [ { "Category": "Document Name", "Description": "The name of the contract", "Answer Format": "Contract Name", "Group": "Group: -", }, { "Category": "Parties", "Description": "The two or more parties who signed the contract", "Answer Format": "Entity or individual names", "Group": "Group: -", }, { "Category": "Agreement Date", "Description": "The date of the contract", "Answer Format": "Date (mm/dd/yyyy)", "Group": "Group: 1", }, { "Category": "Effective Date", "Description": "The date when the contract is effective\u00a0", "Answer Format": "Date (mm/dd/yyyy)", "Group": "Group: 1", }, { "Category": "Expiration Date", "Description": "On what date will the contract's initial term expire?", "Answer Format": "Date (mm/dd/yyyy) / Perpetual", "Group": "Group: 1", }, { "Category": "Renewal Term", "Description": "What is the renewal term after the initial term expires? This includes automatic extensions and unilateral extensions with prior notice.", "Answer Format": "[Successive] number of years/months / Perpetual", "Group": "Group: 1", }, { "Category": "Notice Period to Terminate Renewal", "Description": "What is the notice period required to terminate renewal?", "Answer Format": "Number of days/months/year(s)", "Group": "Group: 1", }, { "Category": "Governing Law", "Description": "Which state/country's law governs the interpretation of the contract?", "Answer Format": "Name of a US State / non-US Province, Country", "Group": "Group: -", }, { "Category": "Most Favored Nation", "Description": "Is there a clause that if a third party gets better terms on the licensing or sale of technology/goods/services described in the contract, the buyer of such technology/goods/services under the contract shall be entitled to those better terms?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Non-Compete", "Description": "Is there a restriction on the ability of a party to compete with the counterparty or operate in a certain geography or business or technology sector?\u00a0", "Answer Format": "Yes/No", "Group": "Group: 2", }, { "Category": "Exclusivity", "Description": "Is there an exclusive dealing\u00a0 commitment with the counterparty? This includes a commitment to procure all \u201crequirements\u201d from one party of certain technology, goods, or services or a prohibition on licensing or selling technology, goods or services to third parties, or a prohibition on\u00a0 collaborating or working with other parties), whether during the contract or\u00a0 after the contract ends (or both).", "Answer Format": "Yes/No", "Group": "Group: 2", }, { "Category": "No-Solicit of Customers", "Description": "Is a party restricted from contracting or soliciting customers or partners of the counterparty, whether during the contract or after the contract ends (or both)?", "Answer Format": "Yes/No", "Group": "Group: 2", }, { "Category": "Competitive Restriction Exception", "Description": "This category includes the exceptions or carveouts to Non-Compete, Exclusivity and No-Solicit of Customers above.", "Answer Format": "Yes/No", "Group": "Group: 2", }, { "Category": "No-Solicit of Employees", "Description": "Is there a restriction on a party\u2019s soliciting or hiring employees and/or contractors from the\u00a0 counterparty, whether during the contract or after the contract ends (or both)?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Non-Disparagement", "Description": "Is there a requirement on a party not to disparage the counterparty?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Termination for Convenience", "Description": "Can a party terminate this\u00a0 contract without cause (solely by giving a notice and allowing a waiting\u00a0 period to expire)?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Rofr/Rofo/Rofn", "Description": "Is there a clause granting one party a right of first refusal, right of first offer or right of first negotiation to purchase, license, market, or distribute equity interest, technology, assets, products or services?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Change of Control", "Description": "Does one party have the right to terminate or is consent or notice required of the counterparty if such party undergoes a change of control, such as a merger, stock sale, transfer of all or substantially all of its assets or business, or assignment by operation of law?", "Answer Format": "Yes/No", "Group": "Group: 3", }, { "Category": "Anti-Assignment", "Description": "Is consent or notice required of a party if the contract is assigned to a third party?", "Answer Format": "Yes/No", "Group": "Group: 3", }, { "Category": "Revenue/Profit Sharing", "Description": "Is one party required to share revenue or profit with the counterparty for any technology, goods, or\u00a0services?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Price Restrictions", "Description": "Is there a restriction on the\u00a0 ability of a party to raise or reduce prices of technology, goods, or\u00a0 services provided?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Minimum Commitment", "Description": "Is there a minimum order size or minimum amount or units per-time period that one party must buy from the counterparty under the contract?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Volume Restriction", "Description": "Is there a fee increase or consent requirement, etc. if one party\u2019s use of the product/services exceeds certain threshold?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "IP Ownership Assignment", "Description": "Does intellectual property created\u00a0 by one party become the property of the counterparty, either per the terms of the contract or upon the occurrence of certain events?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Joint IP Ownership", "Description": "Is there any clause providing for joint or shared ownership of intellectual property between the parties to the contract?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "License Grant", "Description": "Does the contract contain a license granted by one party to its counterparty?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Non-Transferable License", "Description": "Does the contract limit the ability of a party to transfer the license being granted to a third party?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Affiliate License-Licensor", "Description": "Does the contract contain a license grant by affiliates of the licensor or that includes intellectual property of affiliates of the licensor?\u00a0", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Affiliate License-Licensee", "Description": "Does the contract contain a license grant to a licensee (incl. sublicensor) and the affiliates of such licensee/sublicensor?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Unlimited/All-You-Can-Eat-License", "Description": "Is there a clause granting one party an \u201centerprise,\u201d \u201call you can eat\u201d or unlimited usage license?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Irrevocable or Perpetual License", "Description": "Does the contract contain a\u00a0 license grant that is irrevocable or perpetual?", "Answer Format": "Yes/No", "Group": "Group: 4", }, { "Category": "Source Code Escrow", "Description": "Is one party required to deposit its source code into escrow with a third party, which can be released to the counterparty upon the occurrence of certain events (bankruptcy,\u00a0 insolvency, etc.)?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Post-Termination Services", "Description": "Is a party subject to obligations after the termination or expiration of a contract, including any post-termination transition, payment, transfer of IP, wind-down, last-buy, or similar commitments?", "Answer Format": "Yes/No", "Group": "Group: 5", }, { "Category": "Audit Rights", "Description": "Does a party have the right to\u00a0 audit the books, records, or physical locations of the counterparty to ensure compliance with the contract?", "Answer Format": "Yes/No", "Group": "Group: 5", }, { "Category": "Uncapped Liability", "Description": "Is a party\u2019s liability uncapped upon the breach of its obligation in the contract? This also includes uncap liability for a particular type of breach such as IP infringement or breach of confidentiality obligation.", "Answer Format": "Yes/No", "Group": "Group: 6", }, { "Category": "Cap on Liability", "Description": "Does the contract include a cap on liability upon the breach of a party\u2019s obligation? This includes time limitation for the counterparty to bring claims or maximum amount for recovery.", "Answer Format": "Yes/No", "Group": "Group: 6", }, { "Category": "Liquidated Damages", "Description": "Does the contract contain a clause that would award either party liquidated damages for breach or a fee upon the termination of a contract (termination fee)?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Warranty Duration", "Description": "What is the duration of any\u00a0 warranty against defects or errors in technology, products, or services\u00a0 provided under the contract?", "Answer Format": "Number of months or years", "Group": "Group: -", }, { "Category": "Insurance", "Description": "Is there a requirement for insurance that must be maintained by one party for the benefit of the counterparty?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Covenant Not to Sue", "Description": "Is a party restricted from contesting the validity of the counterparty\u2019s ownership of intellectual property or otherwise bringing a claim against the counterparty for matters unrelated to the contract?", "Answer Format": "Yes/No", "Group": "Group: -", }, { "Category": "Third Party Beneficiary", "Description": "Is there a non-contracting party who is a beneficiary to some or all of the clauses in the contract and therefore can enforce its rights against a contracting party?", "Answer Format": "Yes/No", "Group": "Group: -", }, ] NUM_FIELDS_TO_EXTRACT_PER_CONTRACT = 41 # 0.15 is used in the Doc-ETL paper. It should be 0.5 for the actual benchmark. IOU_THRESH = 0.15 # Return the Jaccard similarity between two strings def get_jaccard(label, pred): remove_tokens = [c for c in string.punctuation if c != "/"] for token in remove_tokens: label = label.replace(token, "") pred = pred.replace(token, "") label = label.lower() pred = pred.lower() label = label.replace("/", " ") pred = pred.replace("/", " ") label_words = set(label.split(" ")) pred_words = set(pred.split(" ")) intersection = label_words.intersection(pred_words) union = label_words.union(pred_words) jaccard = len(intersection) / len(union) return jaccard # Find the number of true positives, false positives, and false negatives for each entry # (one field extracted from each contract) by comparing the labels and predictions. # Labels and preds are lists of strings def evaluate_entry(labels, preds, substr_ok): tp, fp, fn = 0, 0, 0 # jaccard similarity expects strings # TODO: This is a hack, ideally, the return type of the preds should be known for idx, pred in enumerate(preds): if not isinstance(pred, str): print(f"Expected string, but got {pred}") preds[idx] = str(pred) # first check if labels is empty if len(labels) == 0: if len(preds) > 0: fp += len(preds) # false positive for each one else: for ans in labels: assert len(ans) > 0 # check if there is a match match_found = False for pred in preds: if substr_ok: is_match = get_jaccard(ans, pred) >= IOU_THRESH or ans in pred else: is_match = get_jaccard(ans, pred) >= IOU_THRESH if is_match: match_found = True if match_found: tp += 1 else: fn += 1 # now also get any fps by looping through preds for pred in preds: # Check if there's a match. if so, don't count (don't want to double count based on the above) # but if there's no match, then this is a false positive. # (Note: we get the true positives in the above loop instead of this loop so that we don't double count # multiple predictions that are matched with the same answer.) match_found = False for ans in labels: assert len(ans) > 0 if substr_ok: is_match = get_jaccard(ans, pred) >= IOU_THRESH or ans in pred else: is_match = get_jaccard(ans, pred) >= IOU_THRESH if is_match: match_found = True if not match_found: fp += 1 return tp, fp, fn # TODO(Siva): This is a temporary fix to handle the case where the preds are empty. def handle_empty_preds(preds): if preds is None or ( # noqa: SIM114 isinstance(preds, str) and (preds == "" or preds == " " or preds == "null" or preds == "None") ): return [] elif isinstance(preds, float) and np.isnan(preds): return [] if not isinstance(preds, (list, np.ndarray)): return [preds] return preds # Compute the precision and recall for the entire dataset. # Each row in the dataframes should correspond to a contract. # The columns should be the extracted fields (categories in cuad_categories). def compute_precision_recall(label_df, preds_df): tp, fp, fn = 0, 0, 0 label_df = label_df.sort_values("contract_id").reset_index(drop=True) preds_df = preds_df.sort_values("contract_id").reset_index(drop=True) assert label_df.shape == preds_df.shape, ( f"Label and prediction dataframes have different shapes, label shape: {label_df.shape} vs preds shape {preds_df.shape}" ) categories = [category["Category"] for category in cuad_categories] for label_row, pred_row in zip(label_df.iterrows(), preds_df.iterrows()): assert label_row[1]["contract_id"] == pred_row[1]["contract_id"], ( f"IDs do not match. label id: {label_row[1]['contract_id']} vs pred id: {pred_row[1]['contract_id']}" ) for category in categories: substr_ok = "Parties" in category labels = label_row[1][category] assert isinstance(labels, list) preds = pred_row[1][category] preds = handle_empty_preds(preds) entry_tp, entry_fp, entry_fn = evaluate_entry(labels, preds, substr_ok) tp += entry_tp fp += entry_fp fn += entry_fn precision = tp / (tp + fp) if tp + fp > 0 else np.nan recall = tp / (tp + fn) if tp + fn > 0 else np.nan return precision, recall class CUADDataset(pz.IterDataset): def __init__(self, num_contracts: int = 1, split: str = "train", seed: int=42): self.num_contracts = num_contracts self.split = split self.seed = seed input_cols = [ {"name": "contract_id", "type": str, "desc": "The id of the the contract to be analyzed"}, {"name": "title", "type": str, "desc": "The title of the the contract to be analyzed"}, {"name": "contract", "type": str, "desc": "The content of the the contract to be analyzed"}, ] super().__init__(id=f"cuad-{split}", schema=input_cols) # convert the dataset into a list of dictionaries where each row is for a single contract include_labels = split == "train" dataset = datasets.load_dataset("theatticusproject/cuad-qa")[split] self.dataset = self._construct_dataset(dataset, num_contracts, seed, include_labels) def _construct_dataset(self, dataset, num_contracts, seed: int=42, include_labels: bool=False): # get the set of unique contract titles; to ensure the order of the contracts is # preserved, we use a list rather than using python's set() contract_titles = [] for row in dataset: if row["title"] not in contract_titles: contract_titles.append(row["title"]) # shuffle the contracts for the given seed rng = np.random.default_rng(seed=seed) rng.shuffle(contract_titles) # get the first num_contracts contract_titles = contract_titles[:num_contracts] # construct the dataset one contract at a time new_dataset = [] for title in contract_titles: # get the rows for this contract contract_rows = [row for row in dataset if row["title"] == title] # construct the contract; we get the contract_id and contract text from the first row contract = { "contract_id": contract_rows[0]["id"], "title": title, "contract": contract_rows[0]["context"], } # for train / validation data, add the labels if include_labels: contract = {"fields": contract} # add the labels category_names = list(map(lambda category: category["Category"], cuad_categories)) contract["labels"] = {category: [] for category in category_names} contract["score_fn"] = {category: None for category in category_names} for row in contract_rows: category_name = row["id"].split("__")[-1].split("_")[0].strip() category_name = category_name.replace(" For ", " for ") category_name = category_name.replace(" Of ", " of ") category_name = category_name.replace(" On ", " on ") category_name = category_name.replace(" Or ", " or ") category_name = category_name.replace(" To ", " to ") category_name = category_name.replace("Ip", "IP") assert category_name in category_names, f"Unknown category {category_name}" contract["labels"][category_name].extend(row["answers"]["text"]) def score_fn(preds, labels, category_name): preds = handle_empty_preds(preds) entry_tp, _, entry_fn = evaluate_entry(labels, preds, substr_ok=True) if category_name == "Parties" else evaluate_entry(labels, preds, substr_ok=False) score = None if len(labels) > 0: # noqa: SIM108 score = entry_tp / (entry_tp + entry_fn) else: score = 1.0 if len(preds) == 0 else 0.0 return score contract["score_fn"][category_name] = partial(score_fn, category_name=category_name) # add the rows to the dataset new_dataset.append(contract) return new_dataset def __len__(self): return self.num_contracts def __getitem__(self, idx: int): return self.dataset[idx] def get_label_df(self): full_dataset = datasets.load_dataset("theatticusproject/cuad-qa")[self.split] label_dataset = self._construct_dataset(full_dataset, self.num_contracts, self.seed, True) final_label_dataset = [] for entry in label_dataset: row = {} row["contract_id"] = entry["fields"]["contract_id"] row["title"] = entry["fields"]["title"] row["contract"] = entry["fields"]["contract"] row = {**row, **entry["labels"]} final_label_dataset.append(row) return pd.DataFrame(final_label_dataset) def parse_arguments(): parser = argparse.ArgumentParser(description="Run CUAD demo") parser.add_argument("--mode", type=str, help="one-convert or separate-converts", default="one-convert") parser.add_argument("--test", type=str, help="test time compute active or inactive", default="active") parser.add_argument("--verbose", default=False, action="store_true", help="Print verbose output") return parser.parse_args() def build_cuad_query(dataset, mode): assert mode in ["one-convert", "separate-converts"] if mode == "one-convert": cols = [] for category in cuad_categories: desc = ( f"Extract the text spans (if they exist) from the contract corresponding to {category['Description']}" ) cols.append({"name": category["Category"], "type": list[str], "desc": desc}) desc = "Extract the text spans (if they exist) from the contract." dataset = dataset.sem_add_columns(cols, desc=desc, depends_on=["contract"]) elif mode == "separate-converts": for category in cuad_categories: desc = ( f"Extract the text spans (if they exist) from the contract corresponding to {category['Description']}" ) dataset = dataset.sem_add_columns( [{"name": category["Category"], "type": list[str], "desc": desc}], desc=category["Description"], depends_on=["contract"], ) return dataset def main(): if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") args = parse_arguments() # create directory for profiling data os.makedirs("opt-profiling-data", exist_ok=True) # Create a data reader for the CUAD dataset dataset = CUADDataset(split="test", num_contracts=1) train_dataset = CUADDataset(split="train", num_contracts=5) print("Created data reader") # Build and run the CUAD query query = build_cuad_query(dataset, args.mode) print("Built query; Starting query execution") execution_strategy = "parallel" sentinel_execution_strategy = "all" optimizer_strategy = "pareto" seed = 0 exp_name = f"cuad-priors-{optimizer_strategy}-seed{seed}" config = pz.QueryProcessorConfig( verbose=False, optimizer_strategy=optimizer_strategy, sentinel_execution_strategy=sentinel_execution_strategy, execution_strategy=execution_strategy, max_workers=64, available_models=[ Model.GPT_4o, Model.GPT_4o_MINI, # Model.LLAMA3_2_3B, Model.LLAMA3_1_8B, Model.LLAMA3_3_70B, # Model.LLAMA3_2_90B_V, # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments` # Model.DEEPSEEK_V3, Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B, ], allow_bonded_query=True, allow_critic=True, allow_mixtures=True, allow_rag_reduction=True, progress=True, k=-1, j=-1, sample_budget=1014*5, seed=seed, exp_name=exp_name, ) data_record_collection = query.optimize_and_run(config=config, train_dataset=train_dataset, validator=pz.Validator()) print("Query execution completed") # save statistics execution_stats_dict = data_record_collection.execution_stats.to_json() with open(f"priors-data/{exp_name}-stats.json", "w") as f: json.dump(execution_stats_dict, f) if __name__ == "__main__": main() ================================================ FILE: abacus-research/helper-scripts/mmqa-baseline.py ================================================ import argparse import json import os import string import time import numpy as np from openai import OpenAI from palimpzest.constants import Cardinality, Model from palimpzest.query.generators.generators import get_json_from_answer def f1(preds: list | None, targets: list): if preds is None or len(targets) == 0: return 0.0 tp, fp, fn = 0, 0, 0 try: # compute recall of retrieved ids and return preds = [str(pred).lower() for pred in preds] targets = [str(target).lower() for target in targets] remove_tokens = [c for c in string.punctuation if c != "/"] for token in remove_tokens: preds = [pred.replace(token, "") for pred in preds] targets = [target.replace(token, "") for target in targets] for pred in preds: if pred in targets: tp += 1 else: fp += 1 for target in targets: if target not in preds: fn += 1 # compute overall f1 score and return recall = tp / (tp + fn) if (tp + fn) > 0 else 0 precision = tp / (tp + fp) if (tp + fp) > 0 else 0 f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 return f1 except Exception: os.makedirs("mmqa-recall-eval-errors", exist_ok=True) ts = time.time() with open(f"mmqa-recall-eval-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 if __name__ == "__main__": # parse arguments parser = argparse.ArgumentParser(description="Run a simple demo") parser.add_argument( "--seed", default=42, type=int, help="Seed used to initialize RNG for MAB sampling algorithm", ) args = parser.parse_args() # create directory for profiling data os.makedirs("opt-profiling-data", exist_ok=True) seed = args.seed print(f"Running with seed: {seed}") # start time for processing start_time = time.time() # read the appropriate dataset dataset = [] with open("data/MMQA_dev.jsonl") as f: for line in f: dict_line = json.loads(line) if "image" in dict_line["metadata"]["modalities"] and len(dict_line["metadata"]["modalities"]) > 1: dataset.append(dict_line) # shuffle the questions for the given seed rng = np.random.default_rng(seed=seed) rng.shuffle(dataset) # trim to number of samples dataset = dataset[:100] # construct the prompt prompt = """You are an intelligent AI assistant designed to answer questions. Please answer the following question to the best of your ability based on your prior knowledge. Return your answer as a JSON list of strings. Do not include any additional context or an explanation in your answer, simply list the entities asked for by the question: QUESTION: {question} ANSWER: """ # iterate over the dataset and generate answers model_name = "gpt-4o-mini-2024-07-18" preds, total_cost = [], 0.0 for idx, entry in enumerate(dataset): print(f"Processing entry {idx}") formatted_prompt = prompt.format(question=entry["question"]) client = OpenAI() payload = { "model": model_name, "temperature": 0.0, "messages": [{"role": "user", "content": formatted_prompt}], } completion = client.chat.completions.create(**payload) # compute total cost model = Model(model_name) usd_per_input_token = model.get_usd_per_input_token() usd_per_output_token = model.get_usd_per_output_token() input_tokens = completion.usage.prompt_tokens output_tokens = completion.usage.completion_tokens total_cost += input_tokens * usd_per_input_token + output_tokens * usd_per_output_token # extract answer completion_text = completion.choices[0].message.content try: answer = get_json_from_answer(completion_text, Model.GPT_4o_MINI, Cardinality.ONE_TO_MANY) except Exception: answer = [completion_text] preds.append(answer) # get total time total_time = time.time() - start_time # score the output scores = [] for pred, entry in zip(preds, dataset): answers = entry["answers"] answer = [ans["answer"] for ans in answers] f1_score = f1(pred, answer) scores.append(f1_score) # create final stats dict stats = {} stats["total_time"] = total_time stats["total_cost"] = total_cost stats["f1"] = np.mean(scores) with open(f'opt-profiling-data/mmqa-baseline-seed-{seed}-stats.json', 'w') as f: json.dump(stats, f) print(stats) ================================================ FILE: abacus-research/helper-scripts/mmqa-gen-image-index.py ================================================ import json import os import chromadb import chromadb.utils.embedding_functions as embedding_functions import numpy as np from PIL import Image from sentence_transformers import SentenceTransformer from tqdm import tqdm CORRUPTED_IMAGE_IDS = [ "17ae0616ac745e70781203267f3a382d", "bf201cbbd058ef51aef89b1be4158c2a", "ef457a7b3ab437cd78ab9f82dc083048", "225c3db49d60b5ef30ed0bfc649ebf78", "b413cc1dc4969dcbe4cb6a55c0f2e359", "e81b2acfd792b171389c8f47a0e14504", ] # NOTE: this script is meant to be run from the root of the repository if __name__ == "__main__": # load CLIP model model = SentenceTransformer("clip-ViT-B-32") # load image metadata image_filepaths, image_ids = [], [] with open("data/MMQA_images.jsonl") as f: possible_endings = {'.JPG', '.png', '.jpeg', '.jpg', '.tif', '.JPEG', '.tiff', '.PNG', '.Jpg', '.gif'} for line in f: dict_line = json.loads(line) image_id = dict_line["id"] # skip corrupted images: if image_id in CORRUPTED_IMAGE_IDS: continue # add image to image_ids image_ids.append(image_id) # find the correct image file for ending in possible_endings: if os.path.exists(f"/ssd1/mdrusso/mmqa-images/{image_id}{ending}"): image_id += ending break image_filepaths.append(f"/ssd1/mdrusso/mmqa-images/{image_id}") # create directory for embeddings os.makedirs("testdata/mmqa-image-embeddings/", exist_ok=True) # generate embeddings in batches of 128 at a time indices = np.linspace(0, len(image_filepaths), len(image_filepaths)//128, dtype=int) total_embeds = len(indices) print(f"Generating {total_embeds} batches of embeddings...") gen_indices = [] for iter_idx, start_idx in tqdm(enumerate(indices), total=total_embeds): # check if embedding needs to be computed end_idx = indices[iter_idx + 1] if iter_idx + 1 < len(indices) else None filename = f"testdata/mmqa-image-embeddings/{start_idx}_{end_idx}.npy" if end_idx is not None and not os.path.exists(filename): # generate embeddings batch_fps = image_filepaths[start_idx:end_idx] batch_images = [Image.open(fp) for fp in batch_fps] embeddings = model.encode(batch_images) # save embeddings to disk with open(filename, "wb") as f: np.save(f, embeddings) gen_indices.append((start_idx, end_idx)) print("Done generating embeddings.") # initialize chroma client chroma_client = chromadb.PersistentClient(".chroma-mmqa") # initialize embedding function sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction( model_name="clip-ViT-B-32" ) # create a collection collection = chroma_client.get_or_create_collection( name="mmqa-images", embedding_function=sentence_transformer_ef, metadata={"hnsw:space": "cosine"}, ) # insert documents in batches total_inserts = len(gen_indices) print(f"Inserting {total_inserts} batches into the collection...") for start_idx, end_idx in tqdm(gen_indices, total=total_inserts): embeddings = np.load(f"testdata/mmqa-image-embeddings/{start_idx}_{end_idx}.npy") collection.add( documents=[os.path.basename(fp) for fp in image_filepaths[start_idx:end_idx]], embeddings=embeddings.tolist(), ids=image_ids[start_idx:end_idx], ) ================================================ FILE: abacus-research/helper-scripts/mmqa-gen-image-title-index.py ================================================ import json import os import time import chromadb import chromadb.utils.embedding_functions as embedding_functions import numpy as np from openai import OpenAI from tqdm import tqdm # NOTE: this script is meant to be run from the root of the repository if __name__ == "__main__": # initialize openai client openai_client = OpenAI() # load image metadata image_title_set = set() image_titles, image_ids = [], [] with open("data/MMQA_images.jsonl") as f: for line in f: dict_line = json.loads(line) image_title = dict_line["title"] if image_title == "": image_title = dict_line["url"] if image_title not in image_title_set: image_titles.append(image_title) image_title_set.add(image_title) else: idx = 1 while f"{image_title} ({idx})" in image_title_set: idx += 1 image_title = f"{image_title} ({idx})" image_titles.append(image_title) image_title_set.add(image_title) image_ids.append(dict_line["id"]) # create directory for embeddings os.makedirs("testdata/mmqa-image-title-embeddings/", exist_ok=True) # generate embeddings in batches of 1000 at a time indices = np.linspace(0, len(image_titles), len(image_titles)//1000, dtype=int) total_embeds = len(indices) print(f"Generating {total_embeds} batches of embeddings...") gen_indices = [] for iter_idx, start_idx in tqdm(enumerate(indices), total=total_embeds): # check if embedding needs to be computed end_idx = indices[iter_idx + 1] if iter_idx + 1 < len(indices) else None filename = f"testdata/mmqa-image-title-embeddings/{start_idx}_{end_idx}.npy" if end_idx is not None and not os.path.exists(filename): # generate embeddings batch = image_titles[start_idx:end_idx] resp = openai_client.embeddings.create(input=batch, model="text-embedding-3-small") embeddings = [item.embedding for item in resp.data] # save embeddings to disk with open(filename, "wb") as f: np.save(f, np.array(embeddings)) gen_indices.append((start_idx, end_idx)) time.sleep(1) print("Done generating embeddings.") # initialize chroma client chroma_client = chromadb.PersistentClient(".chroma-mmqa") # initialize embedding function openai_ef = embedding_functions.OpenAIEmbeddingFunction( api_key=os.environ["OPENAI_API_KEY"], model_name="text-embedding-3-small" ) # create a collection collection = chroma_client.get_or_create_collection( name="mmqa-image-titles", embedding_function=openai_ef, metadata={"hnsw:space": "cosine"}, ) # insert documents in batches total_inserts = len(gen_indices) print(f"Inserting {total_inserts} batches into the collection...") for start_idx, end_idx in tqdm(gen_indices, total=total_inserts): embeddings = np.load(f"testdata/mmqa-image-title-embeddings/{start_idx}_{end_idx}.npy") collection.add( documents=image_titles[start_idx:end_idx], embeddings=embeddings.tolist(), ids=image_ids[start_idx:end_idx], ) ================================================ FILE: abacus-research/helper-scripts/mmqa-gen-table-index.py ================================================ import json import os import time import chromadb import chromadb.utils.embedding_functions as embedding_functions import numpy as np from openai import OpenAI from tqdm import tqdm # NOTE: this script is meant to be run from the root of the repository if __name__ == "__main__": # initialize openai client openai_client = OpenAI() # load table texts table_texts, table_ids = [], [] with open("data/MMQA_tables.jsonl") as f: for line in f: dict_line = json.loads(line) # get page title and table name page_title = dict_line["title"] table_name = dict_line["table"]["table_name"] # get table column names and empty column indices table_header = dict_line["table"]["header"] column_names = [col["column_name"] for col in table_header if col["column_name"] != ""] empty_col_indices = set([idx for idx, col in enumerate(table_header) if col["column_name"] == ""]) # create string for table data text = f"{page_title}: {table_name}\n\n{','.join(column_names)}\n" # parse table row data table_rows = dict_line["table"]["table_rows"] for row in table_rows: row_data = [] for idx, cell in enumerate(row): if idx in empty_col_indices: continue row_data.append(cell["text"]) text += ",".join(row_data) + "\n" table_texts.append(text) table_ids.append(dict_line["id"]) # create directory for embeddings os.makedirs("testdata/mmqa-table-embeddings/", exist_ok=True) # generate embeddings in batches of 1000 at a time indices = np.linspace(0, len(table_texts), len(table_texts)//1000, dtype=int) total_embeds = len(indices) print(f"Generating {total_embeds} batches of embeddings...") gen_indices = [] for iter_idx, start_idx in tqdm(enumerate(indices), total=total_embeds): # check if embedding needs to be computed end_idx = indices[iter_idx + 1] if iter_idx + 1 < len(indices) else None filename = f"testdata/mmqa-table-embeddings/{start_idx}_{end_idx}.npy" if end_idx is not None and not os.path.exists(filename): # generate embeddings batch = table_texts[start_idx:end_idx] resp = openai_client.embeddings.create(input=batch, model="text-embedding-3-small") embeddings = [item.embedding for item in resp.data] # save embeddings to disk with open(filename, "wb") as f: np.save(f, np.array(embeddings)) gen_indices.append((start_idx, end_idx)) time.sleep(1) print("Done generating embeddings.") # initialize chroma client chroma_client = chromadb.PersistentClient(".chroma-mmqa") # initialize embedding function openai_ef = embedding_functions.OpenAIEmbeddingFunction( api_key=os.environ["OPENAI_API_KEY"], model_name="text-embedding-3-small" ) # create a collection collection = chroma_client.get_or_create_collection( name="mmqa-tables", embedding_function=openai_ef, metadata={"hnsw:space": "cosine"}, ) # insert documents in batches total_inserts = len(gen_indices) print(f"Inserting {total_inserts} batches into the collection...") for start_idx, end_idx in tqdm(gen_indices, total=total_inserts): embeddings = np.load(f"testdata/mmqa-table-embeddings/{start_idx}_{end_idx}.npy") collection.add( documents=table_texts[start_idx:end_idx], embeddings=embeddings.tolist(), ids=table_ids[start_idx:end_idx], ) ================================================ FILE: abacus-research/helper-scripts/mmqa-gen-text-index.py ================================================ import json import os import time import chromadb import chromadb.utils.embedding_functions as embedding_functions import numpy as np from openai import OpenAI from tqdm import tqdm # NOTE: this script is meant to be run from the root of the repository if __name__ == "__main__": # initialize openai client openai_client = OpenAI() # load texts texts, text_ids = [], [] with open("data/MMQA_texts.jsonl") as f: for line in f: dict_line = json.loads(line) title = dict_line["title"] text = dict_line["text"] texts.append(f"{title}: {text}") text_ids.append(dict_line["id"]) # create directory for embeddings os.makedirs("testdata/mmqa-text-embeddings/", exist_ok=True) # generate embeddings in batches of 1000 at a time indices = np.linspace(0, len(texts), len(texts)//1000, dtype=int) total_embeds = len(indices) print(f"Generating {total_embeds} batches of embeddings...") gen_indices = [] for iter_idx, start_idx in tqdm(enumerate(indices), total=total_embeds): # check if embedding needs to be computed end_idx = indices[iter_idx + 1] if iter_idx + 1 < len(indices) else None filename = f"testdata/mmqa-text-embeddings/{start_idx}_{end_idx}.npy" if end_idx is not None and not os.path.exists(filename): # generate embeddings batch = texts[start_idx:end_idx] resp = openai_client.embeddings.create(input=batch, model="text-embedding-3-small") embeddings = [item.embedding for item in resp.data] # save embeddings to disk with open(filename, "wb") as f: np.save(f, np.array(embeddings)) gen_indices.append((start_idx, end_idx)) time.sleep(1) print("Done generating embeddings.") # initialize chroma client chroma_client = chromadb.PersistentClient(".chroma-mmqa") # initialize embedding function openai_ef = embedding_functions.OpenAIEmbeddingFunction( api_key=os.environ["OPENAI_API_KEY"], model_name="text-embedding-3-small" ) # create a collection collection = chroma_client.get_or_create_collection( name="mmqa-texts", embedding_function=openai_ef, metadata={"hnsw:space": "cosine"}, ) # insert documents in batches total_inserts = len(gen_indices) print(f"Inserting {total_inserts} batches into the collection...") for start_idx, end_idx in tqdm(gen_indices, total=total_inserts): embeddings = np.load(f"testdata/mmqa-text-embeddings/{start_idx}_{end_idx}.npy") collection.add( documents=texts[start_idx:end_idx], embeddings=embeddings.tolist(), ids=text_ids[start_idx:end_idx], ) ================================================ FILE: abacus-research/mmqa-complex-demo.py ================================================ import argparse import base64 import json import os import string import time import numpy as np import regex as re import palimpzest as pz from palimpzest.constants import Model from palimpzest.core.lib.schemas import ImageBase64 CORRUPTED_IMAGE_IDS = [ "17ae0616ac745e70781203267f3a382d", "bf201cbbd058ef51aef89b1be4158c2a", "ef457a7b3ab437cd78ab9f82dc083048", "225c3db49d60b5ef30ed0bfc649ebf78", "b413cc1dc4969dcbe4cb6a55c0f2e359", "e81b2acfd792b171389c8f47a0e14504", ] mmqa_entry_cols = [ {"name": "qid", "type": str, "desc": "The id of the MMQA question"}, {"name": "question", "type": str, "desc": "The question which needs to be answered"}, ] mmqa_text_search_cols = [ {"name": "text_search_string", "type": str, "desc": "A string used to search for relevant text snippets."}, ] mmqa_table_search_cols = [ {"name": "table_search_string", "type": str, "desc": "A string used to search for relevant tables."}, ] mmqa_image_search_cols = [ {"name": "image_search_string", "type": str, "desc": "A string used to search for relevant images."}, ] mmqa_text_cols = [ {"name": "text_id", "type": str, "desc": "The id for the given text snippet."}, {"name": "text", "type": str, "desc": "A text snippet which may or may not support the question."}, ] mmqa_table_cols = [ {"name": "table_id", "type": str, "desc": "The id for the given table."}, {"name": "table", "type": str, "desc": "A table which may or may not support the question."}, ] mmqa_image_cols = [ {"name": "image_id", "type": str, "desc": "The id for the given image."}, {"name": "image", "type": ImageBase64, "desc": "An image which may or may not support the question."}, ] mmqa_answer_cols = [ {"name": "answers", "type": list[str], "desc": "The answer(s) to the question. Answer the question using the relevant information from gathered image(s), text(s), and table(s). Return your answer as a JSON list of strings. Do not include any additional context or an explanation in your answer, simply list the entities asked for by the question"}, ] def get_json_from_answer(answer: str): """ This function parses an LLM response which is supposed to output a JSON object and optimistically searches for the substring containing the JSON object. """ # split off context / excess, which models sometimes output after answer answer = answer.split("Context:")[0] answer = answer.split("# this is the answer")[0] # trim the answer to only include the JSON array if not answer.strip().startswith("["): # Find the start index of the actual JSON string assuming the prefix is followed by the JSON array start_index = answer.find("[") if start_index != -1: # Remove the prefix and any leading characters before the JSON starts answer = answer[start_index:] if not answer.strip().endswith("]"): # Find the end index of the actual JSON string # assuming the suffix is preceded by the JSON object/array end_index = answer.rfind("]") if end_index != -1: # Remove the suffix and any trailing characters after the JSON ends answer = answer[: end_index + 1] # Handle weird escaped values. I am not sure why the model # is returning these, but the JSON parser can't take them answer = answer.replace(r"\_", "_") answer = answer.replace("\\n", "\n") # Remove https and http prefixes to not conflict with comment detection # Handle comments in the JSON response. Use regex from // until end of line answer = re.sub(r"(? dict: """Compute the label for a MMQA question given its entry in the dataset.""" qid_to_labels = {} for entry in self.dataset: # get the answers answers = [answer["answer"] for answer in entry["answers"]] supporting_text_ids = [context["doc_id"] for context in entry["supporting_context"] if context["doc_part"] == "text"] supporting_table_ids = [context["doc_id"] for context in entry["supporting_context"] if context["doc_part"] == "table"] supporting_image_ids = [context["doc_id"] for context in entry["supporting_context"] if context["doc_part"] == "image"] label_dict = { "answers": answers, "supporting_text_ids": supporting_text_ids, "supporting_table_ids": supporting_table_ids, "supporting_image_ids": supporting_image_ids, } qid_to_labels[entry["qid"]] = label_dict return qid_to_labels def recall(self, preds: list | None, targets: list): if preds is None or len(targets) == 0: return 0.0 tp, fn = 0, 0 try: # compute recall of retrieved ids and return preds = [str(pred).lower() for pred in preds] targets = [str(target).lower() for target in targets] remove_tokens = [c for c in string.punctuation if c != "/"] for token in remove_tokens: preds = [pred.replace(token, "") for pred in preds] targets = [target.replace(token, "") for target in targets] for target in targets: if target in preds: tp += 1 else: fn += 1 return tp / (tp + fn) except Exception: os.makedirs("mmqa-recall-eval-errors", exist_ok=True) ts = time.time() with open(f"mmqa-recall-eval-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def f1(self, preds: list | None, targets: list): if preds is None or len(targets) == 0: return 0.0 tp, fp, fn = 0, 0, 0 try: # compute recall of retrieved ids and return preds = [str(pred).lower() for pred in preds] targets = [str(target).lower() for target in targets] remove_tokens = [c for c in string.punctuation if c != "/"] for token in remove_tokens: preds = [pred.replace(token, "") for pred in preds] targets = [target.replace(token, "") for target in targets] for pred in preds: if pred in targets: tp += 1 else: fp += 1 for target in targets: if target not in preds: fn += 1 # compute overall f1 score and return recall = tp / (tp + fn) if (tp + fn) > 0 else 0 precision = tp / (tp + fp) if (tp + fp) > 0 else 0 f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 return f1 except Exception: os.makedirs("mmqa-recall-eval-errors", exist_ok=True) ts = time.time() with open(f"mmqa-recall-eval-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: if "answers" not in fields: return None preds = output.get("answers") targets = self.qid_to_labels[str(input_record["qid"])]["answers"] return self.f1(preds, targets) def join_score_fn(self, condition: str, left_input_record: dict, right_input_record: dict, output: bool) -> float | None: if condition == "The text snippet is relevant to the question based on the text search string.": pred = right_input_record["text_id"] targets = self.qid_to_labels[left_input_record["qid"]]["supporting_text_ids"] return pred in targets and output or pred not in targets and not output elif condition == "The table is relevant to the question based on the table search string.": pred = right_input_record["table_id"] targets = self.qid_to_labels[left_input_record["qid"]]["supporting_table_ids"] return pred in targets and output or pred not in targets and not output elif condition == "The image is relevant to the question based on the image search string.": pred = right_input_record["image_id"] targets = self.qid_to_labels[left_input_record["qid"]]["supporting_image_ids"] return pred in targets and output or pred not in targets and not output else: raise NotImplementedError(f"Validator.join_score_fn not implemented for condition {condition}.") class MMQAQuestionDataset(pz.IterDataset): def __init__(self, dataset: list[dict]): super().__init__(id="mmqa-questions", schema=mmqa_entry_cols) self.dataset = [{"qid": entry["qid"], "question": entry["question"]} for entry in dataset] def __len__(self): return len(self.dataset) def __getitem__(self, idx: int): return self.dataset[idx] class MMQATextDataset(pz.IterDataset): def __init__(self, dataset: list[dict]): super().__init__(id="mmqa-texts", schema=mmqa_text_cols) # construct mapping from text id to text text_id_to_text = {} with open("data/MMQA_texts.jsonl") as f: for line in f: dict_line = json.loads(line) text_id_to_text[dict_line["id"]] = f"{dict_line['title']}: {dict_line['text']}" # construct dataset self.dataset = [] for entry in dataset: for context in entry["supporting_context"]: if context["doc_part"] == "text": text_id = context["doc_id"] text = text_id_to_text[text_id] self.dataset.append({"text_id": text_id, "text": text}) def __len__(self): return len(self.dataset) def __getitem__(self, idx: int): return self.dataset[idx] class MMQATableDataset(pz.IterDataset): def __init__(self, dataset: list[dict]): super().__init__(id="mmqa-tables", schema=mmqa_table_cols) # construct mapping from table id to table string table_id_to_table = {} with open("data/MMQA_tables.jsonl") as f: for line in f: dict_line = json.loads(line) # get page title and table name page_title = dict_line["title"] table_name = dict_line["table"]["table_name"] # get table column names and empty column indices table_header = dict_line["table"]["header"] column_names = [col["column_name"] for col in table_header if col["column_name"] != ""] empty_col_indices = set([idx for idx, col in enumerate(table_header) if col["column_name"] == ""]) # create string for table data text = f"{page_title}: {table_name}\n\n{','.join(column_names)}\n" # parse table row data table_rows = dict_line["table"]["table_rows"] for row in table_rows: row_data = [] for idx, cell in enumerate(row): if idx in empty_col_indices: continue row_data.append(cell["text"]) text += ",".join(row_data) + "\n" table_id_to_table[dict_line["id"]] = text # construct dataset self.dataset = [] for entry in dataset: for context in entry["supporting_context"]: if context["doc_part"] == "table": table_id = context["doc_id"] table = table_id_to_table[table_id] self.dataset.append({"table_id": table_id, "table": table}) def __len__(self): return len(self.dataset) def __getitem__(self, idx: int): return self.dataset[idx] class MMQAImageDataset(pz.IterDataset): def __init__(self, dataset: list[dict]): super().__init__(id="mmqa-images", schema=mmqa_image_cols) # construct mapping from image id to image base64 object image_id_to_image = {} with open("data/MMQA_images.jsonl") as f: possible_endings = {'.JPG', '.png', '.jpeg', '.jpg', '.tif', '.JPEG', '.tiff', '.PNG', '.Jpg', '.gif'} for line in f: dict_line = json.loads(line) image_id = dict_line["id"] # skip corrupted images: if image_id in CORRUPTED_IMAGE_IDS: continue # find the correct image file image_filepath = None for ending in possible_endings: filepath = f"data/final_dataset_images/{image_id}{ending}" if os.path.exists(filepath): image_filepath = filepath break # read the image file and convert to base64 with open(image_filepath, "rb") as f: contents = base64.b64encode(f.read()).decode("utf-8") image_id_to_image[image_id] = contents # construct dataset self.dataset = [] for entry in dataset: for context in entry["supporting_context"]: if context["doc_part"] == "image": image_id = context["doc_id"] image = image_id_to_image[image_id] self.dataset.append({"image_id": image_id, "image": image}) def __len__(self): return len(self.dataset) def __getitem__(self, idx: int): return self.dataset[idx] def get_dataset(split: str, shuffle: bool, seed: int, num_samples: int | None) -> list[str]: dataset = [] with open(f"data/MMQA_{split}.jsonl") as f: for line in f: dict_line = json.loads(line) if "image" in dict_line["metadata"]["modalities"] and len(dict_line["metadata"]["modalities"]) > 1: dataset.append(dict_line) # shuffle the questions for the given seed if shuffle: rng = np.random.default_rng(seed=seed) rng.shuffle(dataset) return dataset if num_samples is None else dataset[:num_samples] def compute_f1(final_df, answers_df): merged_df = final_df.merge(answers_df, on="qid", how="left") tp, fp, fn = 0, 0, 0 for _, row in merged_df.iterrows(): targets = [str(target).lower() for target in row["gt_answers"]] preds = row["answers"] if isinstance(preds, str): try: # convert single quotes to double quotes before parsing for JSON preds = preds.replace("'", '"') # try parsing preds as JSON list and cast everything to str to match targets preds = get_json_from_answer(preds) preds = [str(pred).lower() for pred in preds] except Exception: # if that fails, give it a shot as a singleton answer that the LLM failed to wrap in a list preds = [preds.lower()] remove_tokens = [c for c in string.punctuation if c != "/"] for token in remove_tokens: preds = [pred.replace(token, "") for pred in preds] targets = [target.replace(token, "") for target in targets] elif isinstance(preds, list): preds = [str(pred).lower() for pred in preds] remove_tokens = [c for c in string.punctuation if c != "/"] for token in remove_tokens: preds = [pred.replace(token, "") for pred in preds] targets = [target.replace(token, "") for target in targets] else: preds = [] for pred in preds: if pred in targets: tp += 1 else: fp += 1 for target in targets: if target not in preds: fn += 1 # compute overall f1 score and return recall = tp / (tp + fn) if (tp + fn) > 0 else 0 precision = tp / (tp + fp) if (tp + fp) > 0 else 0 f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 return f1 if __name__ == "__main__": # parse arguments parser = argparse.ArgumentParser(description="Run a simple demo") parser.add_argument("--verbose", default=False, action="store_true", help="Print verbose output") parser.add_argument("--progress", default=False, action="store_true", help="Print progress output") parser.add_argument("--gpt4-mini-only", default=False, action="store_true", help="Use only GPT-4o-mini") parser.add_argument( "--execution-strategy", default="parallel", type=str, help="The plan executor to use. One of sequential, pipelined, parallel", ) parser.add_argument( "--sentinel-execution-strategy", default="mab", type=str, help="The sentinel execution strategy to use. One of mab or random", ) parser.add_argument( "--policy", default="maxquality", type=str, help="One of 'mincost', 'mintime', 'maxquality'", ) parser.add_argument( "--val-examples", default=20, type=int, help="Number of validation examples to sample from", ) parser.add_argument( "--model", default="gpt-4o", type=str, help="One of 'gpt-4o', 'gpt-4o-mini', 'llama'", ) parser.add_argument( "--seed", default=42, type=int, help="Seed used to initialize RNG for MAB sampling algorithm", ) parser.add_argument( "--k", default=6, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--j", default=4, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--sample-budget", default=100, type=int, help="Total sample budget in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--quality", default=None, type=float, help="Quality threshold", ) parser.add_argument( "--exp-name", default=None, type=str, help="The experiment name.", ) args = parser.parse_args() # create directory for profiling data os.makedirs("mmqa-complex-data", exist_ok=True) verbose = args.verbose progress = args.progress seed = args.seed val_examples = args.val_examples k = args.k j = args.j sample_budget = args.sample_budget execution_strategy = args.execution_strategy sentinel_execution_strategy = args.sentinel_execution_strategy exp_name = ( f"mmqa-complex-final-{sentinel_execution_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}" if args.exp_name is None else args.exp_name ) policy = pz.MaxQuality() if args.policy == "mincost": policy = pz.MinCost() elif args.policy == "minlatency": policy = pz.MinTime() elif args.quality is not None and args.policy == "mincostatfixedquality": policy = pz.MinCostAtFixedQuality(min_quality=args.quality) elif args.quality is not None and args.policy == "minlatencyatfixedquality": policy = pz.MinTimeAtFixedQuality(min_quality=args.quality) print(f"USING POLICY: {policy}") if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") # create the train and test dataset train_dataset = get_dataset(split="train", shuffle=True, seed=seed, num_samples=val_examples) test_dataset = get_dataset(split="dev", shuffle=True, seed=seed, num_samples=100) # create validator for MMQA validator = MMQAValidator(train_dataset) # create train datasets for questions, texts, tables, and images train_question_dataset = MMQAQuestionDataset(train_dataset) train_text_dataset = MMQATextDataset(train_dataset) train_table_dataset = MMQATableDataset(train_dataset) train_image_dataset = MMQAImageDataset(train_dataset) train_dataset = { train_question_dataset.id: train_question_dataset, train_text_dataset.id: train_text_dataset, train_table_dataset.id: train_table_dataset, train_image_dataset.id: train_image_dataset, } # construct plan test_question_dataset = MMQAQuestionDataset(test_dataset) print(f"Test Question Dataset: {len(test_question_dataset)}") test_text_dataset = MMQATextDataset(test_dataset) print(f"Text Dataset: {len(test_text_dataset)}") test_table_dataset = MMQATableDataset(test_dataset) print(f"Table Dataset: {len(test_table_dataset)}") test_image_dataset = MMQAImageDataset(test_dataset) print(f"Image Dataset: {len(test_image_dataset)}") text_plan = test_question_dataset.sem_map(mmqa_text_search_cols, depends_on=["question"]) text_plan = text_plan.sem_join( test_text_dataset, condition="The text snippet is relevant to the question based on the text search string.", depends_on=["text_search_string", "text"], how="left", ) text_plan = text_plan.groupby(pz.GroupBySig(["qid", "question", "text_search_string"], agg_funcs=["list", "list"], agg_fields=["text_id", "text"])) text_plan = text_plan.map( udf=lambda record: {"text": "...".join(record["list(text)"]) if record["list(text)"] != [None] else [None], **record}, cols=[{"name": "text", "type": str, "desc": "All relevant text snippets concatenated together."}], ) text_plan = text_plan.project(["qid", "question", "text"]) table_plan = test_question_dataset.sem_map(mmqa_table_search_cols, depends_on=["question"]) table_plan = table_plan.sem_join( test_table_dataset, condition="The table is relevant to the question based on the table search string.", depends_on=["table_search_string", "table"], how="left", ) table_plan = table_plan.groupby(pz.GroupBySig(["qid", "question", "table_search_string"], agg_funcs=["list", "list"], agg_fields=["table_id", "table"])) table_plan = table_plan.map( udf=lambda record: {"table": "\n\n".join(record["list(table)"]) if record["list(table)"] != [None] else [None], **record}, cols=[{"name": "table", "type": str, "desc": "All relevant tables concatenated together."}], ) table_plan = table_plan.project(["qid", "question", "table"]) image_plan = test_question_dataset.sem_map(mmqa_image_search_cols, depends_on=["question"]) image_plan = image_plan.sem_join( test_image_dataset, condition="The image is relevant to the question based on the image search string.", depends_on=["image_search_string", "image"], how="left", ) image_plan = image_plan.groupby(pz.GroupBySig(["qid", "question", "image_search_string"], agg_funcs=["list", "list"], agg_fields=["image_id", "image"])) image_plan = image_plan.map( udf=lambda record: {"image": record["list(image)"] if record["list(image)"] != [None] else [None], **record}, cols=[{"name": "image", "type": list[ImageBase64], "desc": "All relevant images."}], ) image_plan = image_plan.project(["qid", "question", "image"]) plan = text_plan.join(table_plan, on=["qid", "question"]).join(image_plan, on=["qid", "question"]) plan = plan.sem_map(mmqa_answer_cols, depends_on=["question", "text", "table", "image"]) # execute pz plan config = pz.QueryProcessorConfig( policy=policy, optimizer_strategy="pareto", sentinel_execution_strategy=sentinel_execution_strategy, execution_strategy=execution_strategy, use_final_op_quality=True, max_workers=64, verbose=verbose, available_models=[ Model.GPT_4o_MINI, ], allow_bonded_query=True, allow_critic=True, allow_mixtures=True, allow_rag_reduction=True, progress=progress, k=k, j=j, sample_budget=sample_budget, seed=seed, exp_name=exp_name, ) data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator) print(data_record_collection.to_df()) data_record_collection.to_df().to_csv(f"mmqa-complex-data/{exp_name}-output.csv", index=False) # create filepaths for records and stats records_path = f"mmqa-complex-data/{exp_name}-records.json" stats_path = f"mmqa-complex-data/{exp_name}-profiling.json" # save record outputs record_jsons = [] for record in data_record_collection: record_dict = record.to_dict() record_dict = { k: v for k, v in record_dict.items() if k in ["qid", "question", "supporting_text_ids", "supporting_table_ids", "supporting_image_ids", "answers"] } record_jsons.append(record_dict) with open(records_path, "w") as f: json.dump(record_jsons, f) # read the appropriate dataset dataset = [] with open("data/MMQA_dev.jsonl") as f: for line in f: dict_line = json.loads(line) if "image" in dict_line["metadata"]["modalities"] and len(dict_line["metadata"]["modalities"]) > 1: dataset.append(dict_line) # shuffle the questions for the given seed rng = np.random.default_rng(seed=seed) rng.shuffle(dataset) # trim to 100 samples dataset = dataset[:100] answer_dataset = [] for item in dataset: answers = list(map(lambda elt: str(elt["answer"]), item["answers"])) answer_dataset.append({ "qid": item["qid"], "gt_answers": answers }) # construction dataframe import pandas as pd answers_df = pd.DataFrame(answer_dataset) # get final plan str final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0] final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id] # write stats to disk stats_dict = { "f1": compute_f1(data_record_collection.to_df(), answers_df), "optimization_time": data_record_collection.execution_stats.optimization_time, "optimization_cost": data_record_collection.execution_stats.optimization_cost, "plan_execution_time": data_record_collection.execution_stats.plan_execution_time, "plan_execution_cost": data_record_collection.execution_stats.plan_execution_cost, "total_execution_time": data_record_collection.execution_stats.total_execution_time, "total_execution_cost": data_record_collection.execution_stats.total_execution_cost, "plan_str": final_plan_str, } print(f"F1 IS: {stats_dict['f1']}") with open(f"mmqa-complex-data/{exp_name}-stats.json", "w") as f: json.dump(stats_dict, f) ================================================ FILE: abacus-research/mmqa-demo.py ================================================ import argparse import base64 import json import os import string import time import chromadb import numpy as np import regex as re from chromadb.utils.embedding_functions import ( SentenceTransformerEmbeddingFunction, ) from chromadb.utils.embedding_functions.openai_embedding_function import ( OpenAIEmbeddingFunction, ) import palimpzest as pz from palimpzest.constants import Model from palimpzest.core.lib.schemas import ImageBase64 mmqa_entry_cols = [ {"name": "qid", "type": str, "desc": "The id of the MMQA question"}, {"name": "question", "type": str, "desc": "The question which needs to be answered"}, ] mmqa_text_cols = [ {"name": "supporting_text_ids", "type": list[str], "desc": "A list of text ids for text snippets which may support the question."}, {"name": "supporting_texts", "type": list[str], "desc": "A list of text snippets which may support the question."}, ] mmqa_table_cols = [ {"name": "supporting_table_ids", "type": list[str], "desc": "A list of table ids for tables which may support the question."}, {"name": "supporting_tables", "type": list[str], "desc": "A list of tables which may support the question."}, ] mmqa_image_cols = [ {"name": "supporting_image_ids", "type": list[str], "desc": "A list of image ids whose images may support the question."}, {"name": "supporting_images", "type": list[ImageBase64], "desc": "A list of images which may support the question."}, ] mmqa_answer_cols = [ {"name": "answers", "type": list[str], "desc": "The answer(s) to the question. Answer the question using the relevant information from gathered image(s), text(s), and table(s). Do not include any additional context or an explanation in your final list, simply list the entities asked for by the question"}, ] # Return your answer as a JSON list of strings. def get_json_from_answer(answer: str): """ This function parses an LLM response which is supposed to output a JSON object and optimistically searches for the substring containing the JSON object. """ # split off context / excess, which models sometimes output after answer answer = answer.split("Context:")[0] answer = answer.split("# this is the answer")[0] # trim the answer to only include the JSON array if not answer.strip().startswith("["): # Find the start index of the actual JSON string assuming the prefix is followed by the JSON array start_index = answer.find("[") if start_index != -1: # Remove the prefix and any leading characters before the JSON starts answer = answer[start_index:] if not answer.strip().endswith("]"): # Find the end index of the actual JSON string # assuming the suffix is preceded by the JSON object/array end_index = answer.rfind("]") if end_index != -1: # Remove the suffix and any trailing characters after the JSON ends answer = answer[: end_index + 1] # Handle weird escaped values. I am not sure why the model # is returning these, but the JSON parser can't take them answer = answer.replace(r"\_", "_") answer = answer.replace("\\n", "\n") # Remove https and http prefixes to not conflict with comment detection # Handle comments in the JSON response. Use regex from // until end of line answer = re.sub(r"(? 1: dataset.append(dict_line) # shuffle the questions for the given seed if shuffle: rng = np.random.default_rng(seed=seed) rng.shuffle(dataset) # trim to number of samples self.dataset = dataset[:num_samples] self.num_samples = num_samples self.shuffle = shuffle self.seed = seed # compute qid to label mapping self.qid_to_labels = self._compute_qid_to_labels() def _compute_qid_to_labels(self) -> dict: """Compute the label for a MMQA question given its entry in the dataset.""" qid_to_labels = {} for entry in self.dataset: # get the answers answers = [answer["answer"] for answer in entry["answers"]] supporting_text_doc_ids = [context["doc_id"] for context in entry["supporting_context"] if context["doc_part"] == "text"] supporting_table_doc_ids = [context["doc_id"] for context in entry["supporting_context"] if context["doc_part"] == "table"] supporting_image_doc_ids = [context["doc_id"] for context in entry["supporting_context"] if context["doc_part"] == "image"] # NOTE: inside the optimizer, our qualities will effectively be divided by two, # because we are not providing a label for supporting texts, tables, and images, # however this should be okay b/c it will affect all records equally label_dict = { "answers": answers, "supporting_text_ids": supporting_text_doc_ids, "supporting_table_ids": supporting_table_doc_ids, "supporting_image_ids": supporting_image_doc_ids, "supporting_texts": [], "supporting_tables": [], "supporting_images": [], } qid_to_labels[entry["qid"]] = label_dict return qid_to_labels def recall(self, preds: list | None, targets: list): if preds is None or len(targets) == 0: return 0.0 tp, fn = 0, 0 try: if isinstance(preds, list) and len(preds) > 0 and isinstance(preds[0], list): preds = preds[0] # compute recall of retrieved ids and return preds = [str(pred).lower() for pred in preds] targets = [str(target).lower() for target in targets] remove_tokens = [c for c in string.punctuation if c != "/"] for token in remove_tokens: preds = [pred.replace(token, "") for pred in preds] targets = [target.replace(token, "") for target in targets] for target in targets: if target in preds: tp += 1 else: fn += 1 return tp / (tp + fn) except Exception: os.makedirs("mmqa-recall-eval-errors", exist_ok=True) ts = time.time() with open(f"mmqa-recall-eval-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def f1(self, preds: list | None, targets: list): if preds is None or len(targets) == 0: return 0.0 tp, fp, fn = 0, 0, 0 try: if isinstance(preds, list) and len(preds) > 0 and isinstance(preds[0], list): preds = preds[0] # compute recall of retrieved ids and return preds = [str(pred).lower() for pred in preds] targets = [str(target).lower() for target in targets] remove_tokens = [c for c in string.punctuation if c != "/"] for token in remove_tokens: preds = [pred.replace(token, "") for pred in preds] targets = [target.replace(token, "") for target in targets] for pred in preds: if pred in targets: tp += 1 else: fp += 1 for target in targets: if target not in preds: fn += 1 # compute overall f1 score and return recall = tp / (tp + fn) if (tp + fn) > 0 else 0 precision = tp / (tp + fp) if (tp + fp) > 0 else 0 f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 return f1 except Exception: os.makedirs("mmqa-recall-eval-errors", exist_ok=True) ts = time.time() with open(f"mmqa-recall-eval-errors/error-{ts}.txt", "w") as f: f.write(str(preds)) return 0.0 def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: preds = output.get("answers") targets = self.qid_to_labels[str(input_record["qid"])]["answers"] return self.f1(preds, targets) def topk_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: if "supporting_text_ids" in fields: preds = output.get("supporting_text_ids") targets = self.qid_to_labels[str(input_record["qid"])]["supporting_text_ids"] return self.recall(preds, targets) elif "supporting_table_ids" in fields: preds = output.get("supporting_table_ids") targets = self.qid_to_labels[str(input_record["qid"])]["supporting_table_ids"] return self.recall(preds, targets) elif "supporting_image_ids" in fields: preds = output.get("supporting_image_ids") targets = self.qid_to_labels[str(input_record["qid"])]["supporting_image_ids"] return self.recall(preds, targets) else: raise NotImplementedError(f"Validator.topk_score_fn not implemented for fields {fields}.") class MMQADataset(pz.IterDataset): def __init__( self, num_samples: int = 5, split: str = "dev", shuffle: bool = False, seed: int = 42, ): super().__init__(id="mmqa", schema=mmqa_entry_cols) # read the appropriate dataset dataset = [] with open(f"data/MMQA_{split}.jsonl") as f: for line in f: dict_line = json.loads(line) if "image" in dict_line["metadata"]["modalities"] and len(dict_line["metadata"]["modalities"]) > 1: dataset.append(dict_line) # shuffle the questions for the given seed if shuffle: rng = np.random.default_rng(seed=seed) rng.shuffle(dataset) # trim to number of samples self.dataset = dataset[:num_samples] self.num_samples = num_samples self.shuffle = shuffle self.seed = seed self.split = split def __len__(self): return len(self.dataset) def __getitem__(self, idx: int): # get entry entry = self.dataset[idx] # get input fields qid = entry["qid"] question = entry["question"] # create item with fields item = {"qid": qid, "question": question} return item def compute_f1(final_df, answers_df): merged_df = final_df.merge(answers_df, on="qid", how="left") tp, fp, fn = 0, 0, 0 for _, row in merged_df.iterrows(): targets = [str(target).lower() for target in row["gt_answers"]] preds = row["answers"] if isinstance(preds, str): try: # convert single quotes to double quotes before parsing for JSON preds = preds.replace("'", '"') # try parsing preds as JSON list and cast everything to str to match targets preds = get_json_from_answer(preds) if isinstance(preds, list) and len(preds) > 0 and isinstance(preds[0], list): preds = preds[0] preds = [str(pred).lower() for pred in preds] except Exception: # if that fails, give it a shot as a singleton answer that the LLM failed to wrap in a list preds = [str(preds).lower()] remove_tokens = [c for c in string.punctuation if c != "/"] for token in remove_tokens: preds = [pred.replace(token, "") for pred in preds] targets = [target.replace(token, "") for target in targets] elif isinstance(preds, list): if isinstance(preds, list) and len(preds) > 0 and isinstance(preds[0], list): preds = preds[0] preds = [str(pred).lower() for pred in preds] remove_tokens = [c for c in string.punctuation if c != "/"] for token in remove_tokens: preds = [pred.replace(token, "") for pred in preds] targets = [target.replace(token, "") for target in targets] else: preds = [] for pred in preds: if pred in targets: tp += 1 else: fp += 1 for target in targets: if target not in preds: fn += 1 # compute overall f1 score and return recall = tp / (tp + fn) if (tp + fn) > 0 else 0 precision = tp / (tp + fp) if (tp + fp) > 0 else 0 f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 return f1 if __name__ == "__main__": # parse arguments parser = argparse.ArgumentParser(description="Run a simple demo") parser.add_argument("--verbose", default=False, action="store_true", help="Print verbose output") parser.add_argument("--progress", default=False, action="store_true", help="Print progress output") parser.add_argument("--gpt4-mini-only", default=False, action="store_true", help="Use only GPT-4o-mini") parser.add_argument( "--execution-strategy", default="parallel", type=str, help="The plan executor to use. One of sequential, pipelined, parallel", ) parser.add_argument( "--sentinel-execution-strategy", default="mab", type=str, help="The sentinel execution strategy to use. One of mab or random", ) parser.add_argument( "--policy", default="maxquality", type=str, help="One of 'mincost', 'mintime', 'maxquality'", ) parser.add_argument( "--val-examples", default=20, type=int, help="Number of validation examples to sample from", ) parser.add_argument( "--model", default="gpt-4o", type=str, help="One of 'gpt-4o', 'gpt-4o-mini', 'llama'", ) parser.add_argument( "--seed", default=42, type=int, help="Seed used to initialize RNG for MAB sampling algorithm", ) parser.add_argument( "--k", default=6, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--j", default=4, type=int, help="Number of columns to sample in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--sample-budget", default=100, type=int, help="Total sample budget in Random Sampling or MAB sentinel execution", ) parser.add_argument( "--quality", default=None, type=float, help="Quality threshold", ) parser.add_argument( "--exp-name", default=None, type=str, help="The experiment name.", ) args = parser.parse_args() # create directory for profiling data os.makedirs("opt-profiling-data", exist_ok=True) verbose = args.verbose progress = args.progress seed = args.seed val_examples = args.val_examples k = args.k j = args.j sample_budget = args.sample_budget execution_strategy = args.execution_strategy sentinel_execution_strategy = args.sentinel_execution_strategy exp_name = ( f"mmqa-final-{sentinel_execution_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}" if args.exp_name is None else args.exp_name ) policy = pz.MaxQuality() if args.policy == "mincost": policy = pz.MinCost() elif args.policy == "minlatency": policy = pz.MinTime() elif args.quality is not None and args.policy == "mincostatfixedquality": policy = pz.MinCostAtFixedQuality(min_quality=args.quality) elif args.quality is not None and args.policy == "minlatencyatfixedquality": policy = pz.MinTimeAtFixedQuality(min_quality=args.quality) print(f"USING POLICY: {policy}") if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") # create validator for MMQA validator = MMQAValidator(num_samples=val_examples, shuffle=True, seed=seed) # create datasets for MMQA train_dataset = MMQADataset(split="train", num_samples=val_examples, shuffle=True, seed=seed) train_dataset = {train_dataset.id: train_dataset} # load index [text-embedding-3-small] chroma_client = chromadb.PersistentClient(".chroma-mmqa") openai_ef = OpenAIEmbeddingFunction( api_key=os.environ["OPENAI_API_KEY"], model_name="text-embedding-3-small", ) sentence_transformer_ef = SentenceTransformerEmbeddingFunction( model_name="clip-ViT-B-32" ) text_index = chroma_client.get_collection("mmqa-texts", embedding_function=openai_ef) table_index = chroma_client.get_collection("mmqa-tables", embedding_function=openai_ef) image_index = chroma_client.get_collection("mmqa-images", embedding_function=sentence_transformer_ef) def get_results_and_ids(index: chromadb.Collection, query: list[list[float]], n_results: int, image=False) -> tuple[list[str]]: # execute query with embeddings results = index.query(query, n_results=n_results) # get list of result terms with their cosine similarity scores final_results = [] for query_doc_ids, query_docs, query_distances in zip(results["ids"], results["documents"], results["distances"]): for doc_id, doc, dist in zip(query_doc_ids, query_docs, query_distances): cosine_similarity = 1 - dist final_results.append({"content": doc, "id": doc_id, "similarity": cosine_similarity}) # sort the results by similarity score sorted_results = sorted(final_results, key=lambda result: result["similarity"], reverse=True) # remove duplicates sorted_results_set = set() final_sorted_results, final_sorted_result_ids = [], [] for result in sorted_results: if result["content"] not in sorted_results_set: sorted_results_set.add(result["content"]) final_sorted_results.append(result["content"]) final_sorted_result_ids.append(result["id"]) # return the top-k similar results and generation stats return final_sorted_results[:n_results], final_sorted_result_ids[:n_results] def text_search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]: # execute query with embeddings results, result_ids = get_results_and_ids(index, query, n_results=k) return {"supporting_texts": results, "supporting_text_ids": result_ids} def table_search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]: # execute query with embeddings results, result_ids = get_results_and_ids(index, query, n_results=k) return {"supporting_tables": results, "supporting_table_ids": result_ids} def image_search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]: # limit max number of results to 5 # k = min(k, 5) # execute query with embeddings _, result_ids = get_results_and_ids(index, query, n_results=k, image=True) possible_endings = {'.JPG', '.png', '.jpeg', '.jpg', '.tif', '.JPEG', '.tiff', '.PNG', '.Jpg', '.gif'} results = [] for image_id in result_ids: # find the correct image file for ending in possible_endings: if os.path.exists(f"/ssd1/mdrusso/mmqa-images/{image_id}{ending}"): image_id += ending break # load image from disk with open(f"/ssd1/mdrusso/mmqa-images/{image_id}", "rb") as f: base64_image_str = base64.b64encode(f.read()).decode("utf-8") results.append(base64_image_str) return {"supporting_images": results, "supporting_image_ids": result_ids} # construct plan plan = MMQADataset(split="dev", num_samples=100, shuffle=True, seed=seed) plan = plan.sem_topk( index=text_index, search_func=text_search_func, search_attr="question", output_attrs=mmqa_text_cols, ) plan = plan.sem_topk( index=table_index, search_func=table_search_func, search_attr="question", output_attrs=mmqa_table_cols, ) plan = plan.sem_topk( index=image_index, search_func=image_search_func, search_attr="question", output_attrs=mmqa_image_cols, ) plan = plan.sem_map(mmqa_answer_cols, depends_on=["question", "supporting_texts", "supporting_tables", "supporting_images"]) # execute pz plan config = pz.QueryProcessorConfig( policy=policy, optimizer_strategy="pareto", sentinel_execution_strategy=sentinel_execution_strategy, execution_strategy=execution_strategy, use_final_op_quality=True, max_workers=1, verbose=verbose, available_models=[ Model.GPT_4o_MINI, ], allow_bonded_query=True, allow_critic=True, allow_mixtures=True, allow_rag_reduction=True, progress=progress, k=k, j=j, sample_budget=sample_budget, seed=seed, exp_name=exp_name, ) data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator) print(data_record_collection.to_df()) data_record_collection.to_df().to_csv(f"opt-profiling-data/{exp_name}-output.csv", index=False) # create filepaths for records and stats records_path = f"opt-profiling-data/{exp_name}-records.json" stats_path = f"opt-profiling-data/{exp_name}-profiling.json" # save record outputs record_jsons = [] for record in data_record_collection: record_dict = record.to_dict() record_dict = { k: v for k, v in record_dict.items() if k in ["qid", "question", "supporting_text_ids", "supporting_table_ids", "supporting_image_ids", "answers"] } record_jsons.append(record_dict) with open(records_path, "w") as f: json.dump(record_jsons, f) # read the appropriate dataset dataset = [] with open("data/MMQA_dev.jsonl") as f: for line in f: dict_line = json.loads(line) if "image" in dict_line["metadata"]["modalities"] and len(dict_line["metadata"]["modalities"]) > 1: dataset.append(dict_line) # shuffle the questions for the given seed rng = np.random.default_rng(seed=seed) rng.shuffle(dataset) # trim to 100 samples dataset = dataset[:100] answer_dataset = [] for item in dataset: answers = list(map(lambda elt: str(elt["answer"]), item["answers"])) answer_dataset.append({ "qid": item["qid"], "gt_answers": answers }) # construction dataframe import pandas as pd answers_df = pd.DataFrame(answer_dataset) # get final plan str final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0] final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id] # write stats to disk stats_dict = { "f1": compute_f1(data_record_collection.to_df(), answers_df), "optimization_time": data_record_collection.execution_stats.optimization_time, "optimization_cost": data_record_collection.execution_stats.optimization_cost, "plan_execution_time": data_record_collection.execution_stats.plan_execution_time, "plan_execution_cost": data_record_collection.execution_stats.plan_execution_cost, "total_execution_time": data_record_collection.execution_stats.total_execution_time, "total_execution_cost": data_record_collection.execution_stats.total_execution_cost, "plan_str": final_plan_str, } print(f"F1 IS: {stats_dict['f1']}") with open(f"opt-profiling-data/{exp_name}-stats.json", "w") as f: json.dump(stats_dict, f) ================================================ FILE: abacus-research/run_ablation_study.sh ================================================ #!/bin/bash for seed in {0..9} do for priors in none naive sample do for sentinel in mab random do for opt in pareto greedy do k=6 j=4 if [[ $sentinel -eq "random" ]]; then j=8 fi priors_file="none" if [[ $priors -eq "naive" ]]; then priors_file="cheap-priors.json" elif [[ $priors -eq "sample" ]]; then priors_file="biodex-priors.json" fi exp_name="ablation-${priors}-${sentinel}-${opt}-seed${seed}" FILE="ablation-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- priors: ${priors} (${priors_file}) -- sentinel: ${sentinel} -- k: ${k} -- j: ${j} -- opt: ${opt}" python biodex-ablation.py --priors-file $priors_file --k $k --j $j --sample-budget 150 --optimizer-strategy $opt --sentinel-execution-strategy $sentinel --seed $seed --exp-name $exp_name fi done done done done ================================================ FILE: abacus-research/run_biodex.sh ================================================ #!/bin/bash for seed in {0..9} do echo "Running Seed: ${seed}" exp_name="biodex-final-mab-k6-j4-budget150-seed${seed}" python biodex-demo.py --progress --policy maxquality --val-examples 20 --k 6 --j 4 --sample-budget 150 --seed $seed --exp-name $exp_name --gpt4-mini-only done ================================================ FILE: abacus-research/run_biodex_cascades.sh ================================================ #!/bin/bash for seed in {0..9} do for budget in 150 300 450 do for strategy in "greedy" "pareto" do cost=0.5 k=0 j=0 if [[ $budget -eq 150 ]]; then k=6 j=4 elif [[ $budget -eq 300 ]]; then k=24 j=5 elif [[ $budget -eq 450 ]]; then k=48 j=6 fi # no priors exp_name="biodex-${strategy}-cost${cost}-budget${budget}-k${k}-j${j}-seed${seed}" FILE="pareto-cascades-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- cost: ${cost} -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}" python biodex-pareto-cascades.py --progress --k $k --j $j --sample-budget $budget --optimizer-strategy $strategy --cost $cost --seed $seed --exp-name $exp_name fi # sample priors exp_name="biodex-${strategy}-cost${cost}-with-priors-budget${budget}-k${k}-j${j}-seed${seed}" FILE="pareto-cascades-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- cost: ${cost} -- SAMPLE PRIORS -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}" python biodex-pareto-cascades.py --progress --priors-file biodex-priors-cascades.json --k $k --j $j --sample-budget $budget --optimizer-strategy $strategy --cost $cost --seed $seed --exp-name $exp_name fi # naive priors exp_name="biodex-${strategy}-cost${cost}-cheap-priors-budget${budget}-k${k}-j${j}-seed${seed}" FILE="pareto-cascades-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- cost: ${cost} -- CHEAP PRIORS -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}" python biodex-pareto-cascades.py --progress --priors-file cheap-priors-cascades.json --k $k --j $j --sample-budget $budget --optimizer-strategy $strategy --cost $cost --seed $seed --exp-name $exp_name fi done done done ================================================ FILE: abacus-research/run_biodex_cost_threshold.sh ================================================ #!/bin/bash for cost in 1.0 2.0 4.0 8.0 999.99 do for seed in {0..9} do # set variables budget=450 k=48 j=3 # no priors exp_name="biodex-pareto-cost${cost}-budget${budget}-k${k}-j${j}-seed${seed}" FILE="max-quality-at-cost-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- cost: ${cost} -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}" python biodex-max-quality-at-cost.py --progress --k $k --j $j --sample-budget $budget --cost $cost --seed $seed --exp-name $exp_name fi # sample priors exp_name="biodex-pareto-cost${cost}-with-priors-budget${budget}-k${k}-j${j}-seed${seed}" FILE="max-quality-at-cost-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- cost: ${cost} -- SAMPLE PRIORS -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}" python biodex-max-quality-at-cost.py --progress --priors-file biodex-priors.json --k $k --j $j --sample-budget $budget --cost $cost --seed $seed --exp-name $exp_name fi done done ================================================ FILE: abacus-research/run_biodex_min_cost_latency.sh ================================================ #!/bin/bash for policy in "mincost" "minlatency" do for seed in {0..9} do # set variables budget=150 k=6 j=4 echo "Running Seed: ${seed}" exp_name="biodex-final-${policy}-k6-j4-budget150-seed${seed}" python biodex-demo.py --progress --policy $policy --val-examples 20 --k 6 --j 4 --sample-budget 150 --seed $seed --exp-name $exp_name --gpt4-mini-only done done ================================================ FILE: abacus-research/run_biodex_priors.sh ================================================ #!/bin/bash for sample_budget in 10 20 50 100 do for seed in {0..9} do k=0 j=0 if [[ $sample_budget -eq 10 ]]; then k=2 j=2 elif [[ $sample_budget -eq 20 ]]; then k=2 j=2 elif [[ $sample_budget -eq 50 ]]; then k=3 j=3 elif [[ $sample_budget -eq 100 ]]; then k=4 j=4 fi # run without priors exp_name="biodex-no-priors-k${k}-j${j}-budget${sample_budget}-seed${seed}" FILE="opt-profiling-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- priors: NO PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}" python biodex-demo.py --progress --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name fi # run with sample-based priors exp_name="biodex-with-priors-k${k}-j${j}-budget${sample_budget}-seed${seed}" FILE="opt-profiling-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- priors: WITH PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}" python biodex-demo.py --progress --priors-file biodex-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name fi # run with cheap priors exp_name="biodex-cheap-priors-k${k}-j${j}-budget${sample_budget}-seed${seed}" FILE="opt-profiling-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- priors: CHEAP PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}" python biodex-demo.py --progress --priors-file cheap-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name fi done done ================================================ FILE: abacus-research/run_biodex_priors_constrained.sh ================================================ #!/bin/bash for sample_budget in 10 20 50 100 do for seed in {0..9} do k=0 j=0 if [[ $sample_budget -eq 10 ]]; then k=4 j=1 elif [[ $sample_budget -eq 20 ]]; then k=4 j=1 elif [[ $sample_budget -eq 50 ]]; then k=4 j=2 elif [[ $sample_budget -eq 100 ]]; then k=5 j=3 fi # run without priors exp_name="biodex-no-priors-constrained-k${k}-j${j}-budget${sample_budget}-seed${seed}" FILE="opt-profiling-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- priors: NO PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}" python biodex-demo.py --constrained --progress --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name fi # run with sample-based priors exp_name="biodex-with-priors-constrained-k${k}-j${j}-budget${sample_budget}-seed${seed}" FILE="opt-profiling-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- priors: WITH PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}" python biodex-demo.py --constrained --progress --priors-file biodex-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name fi # run with cheap priors exp_name="biodex-cheap-priors-constrained-k${k}-j${j}-budget${sample_budget}-seed${seed}" FILE="opt-profiling-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- priors: CHEAP PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}" python biodex-demo.py --constrained --progress --priors-file cheap-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name fi done done ================================================ FILE: abacus-research/run_cuad.sh ================================================ #!/bin/bash for seed in {0..9} do policy="maxquality" echo "Running Seed: ${seed} -- policy: ${policy}" exp_name="cuad-${policy}-k6-j4-budget50-seed${seed}" python cuad-demo.py --k 6 --j 4 --sample-budget 50 --seed $seed --exp-name $exp_name --gpt4-mini-only done ================================================ FILE: abacus-research/run_cuad_cost_threshold.sh ================================================ #!/bin/bash for cost in 1.0 2.0 4.0 8.0 999.99 do for seed in {0..9} do # set variables budget=300 k=60 j=5 # no priors exp_name="cuad-pareto-cost${cost}-budget${budget}-k${k}-j${j}-seed${seed}" FILE="max-quality-at-cost-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- cost: ${cost} -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}" python cuad-max-quality-at-cost.py --k $k --j $j --sample-budget $budget --cost $cost --seed $seed --exp-name $exp_name fi # sample priors exp_name="cuad-pareto-cost${cost}-with-priors-budget${budget}-k${k}-j${j}-seed${seed}" FILE="max-quality-at-cost-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- cost: ${cost} -- SAMPLE PRIORS -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}" python cuad-max-quality-at-cost.py --priors-file cuad-priors.json --k $k --j $j --sample-budget $budget --cost $cost --seed $seed --exp-name $exp_name fi done done ================================================ FILE: abacus-research/run_cuad_min_cost_latency.sh ================================================ #!/bin/bash for policy in "mincost" "minlatency" do for seed in {0..9} do echo "Running Seed: ${seed}" exp_name="cuad-final-${policy}-k6-j4-budget50-seed${seed}" python cuad-demo.py --policy $policy --k 6 --j 4 --sample-budget 50 --seed $seed --exp-name $exp_name --gpt4-mini-only done done ================================================ FILE: abacus-research/run_cuad_priors.sh ================================================ #!/bin/bash for sample_budget in 5 10 20 50 do for seed in {0..9} do k=0 j=0 if [[ $sample_budget -eq 5 ]]; then k=2 j=3 elif [[ $sample_budget -eq 10 ]]; then k=3 j=2 elif [[ $sample_budget -eq 20 ]]; then k=3 j=3 elif [[ $sample_budget -eq 50 ]]; then k=6 j=4 fi # run without priors exp_name="cuad-no-priors-k${k}-j${j}-budget${sample_budget}-seed${seed}" FILE="opt-profiling-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- priors: NO PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}" python cuad-demo.py --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name fi # run with sample based priors exp_name="cuad-with-priors-k${k}-j${j}-budget${sample_budget}-seed${seed}" FILE="opt-profiling-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- priors: WITH PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}" python cuad-demo.py --priors-file cuad-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name fi # run with cheap priors exp_name="cuad-cheap-priors-k${k}-j${j}-budget${sample_budget}-seed${seed}" FILE="opt-profiling-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- priors: CHEAP PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}" python cuad-demo.py --priors-file cheap-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name fi done done ================================================ FILE: abacus-research/run_cuad_priors_constrained.sh ================================================ #!/bin/bash for sample_budget in 5 10 20 50 do for seed in {0..9} do k=0 j=0 if [[ $sample_budget -eq 5 ]]; then k=2 j=3 elif [[ $sample_budget -eq 10 ]]; then k=3 j=2 elif [[ $sample_budget -eq 20 ]]; then k=3 j=3 elif [[ $sample_budget -eq 50 ]]; then k=6 j=4 fi # run without priors exp_name="cuad-no-priors-constrained-k${k}-j${j}-budget${sample_budget}-seed${seed}" FILE="opt-profiling-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- priors: NO PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}" python cuad-demo.py --constrained --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name fi # run with sample based priors exp_name="cuad-with-priors-constrained-k${k}-j${j}-budget${sample_budget}-seed${seed}" FILE="opt-profiling-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- priors: WITH PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}" python cuad-demo.py --constrained --priors-file cuad-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name fi # run with cheap priors exp_name="cuad-cheap-priors-constrained-k${k}-j${j}-budget${sample_budget}-seed${seed}" FILE="opt-profiling-data/${exp_name}-metrics.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- priors: CHEAP PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}" python cuad-demo.py --constrained --priors-file cheap-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name fi done done ================================================ FILE: abacus-research/run_mmqa.sh ================================================ #!/bin/bash for seed in {0..9} do echo "Running Seed: ${seed}" exp_name="mmqa-final-mab-k6-j4-budget150-seed${seed}" python mmqa-demo.py --progress --k 6 --j 4 --sample-budget 150 --seed $seed --exp-name $exp_name --gpt4-mini-only done ================================================ FILE: abacus-research/run_mmqa_complex.sh ================================================ #!/bin/bash # for seed in {0..9} # Lotus error'ed on seed 5 and 7, so we limit to these seeds only for a consistent comparison for seed in 0 1 2 3 4 6 8 9 do policy="maxquality" exp_name="mmqa-complex-${policy}-k6-j4-budget350-seed${seed}" FILE="mmqa-complex-data/${exp_name}-stats.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- ${policy}" python mmqa-complex-demo.py --progress --k 6 --j 4 --sample-budget 350 --seed $seed --exp-name $exp_name --gpt4-mini-only fi done ================================================ FILE: abacus-research/run_mmqa_complex_min_cost_latency.sh ================================================ #!/bin/bash # for seed in {0..9} # Lotus error'ed on seed 5 and 7, so we limit to these seeds only for a consistent comparison for seed in 0 1 2 3 4 6 8 9 do policy="mincost" exp_name="mmqa-complex-${policy}-k6-j4-budget350-seed${seed}" FILE="mmqa-complex-data/${exp_name}-stats.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- ${policy}" python mmqa-complex-demo.py --progress --k 6 --j 4 --sample-budget 350 --policy $policy --seed $seed --exp-name $exp_name --gpt4-mini-only fi policy="minlatency" exp_name="mmqa-complex-${policy}-k6-j4-budget350-seed${seed}" FILE="mmqa-complex-data/${exp_name}-stats.json" if [ -f $FILE ]; then echo "Skipping because $FILE exists." else echo "Running Seed: ${seed} -- ${policy}" python mmqa-complex-demo.py --progress --k 6 --j 4 --sample-budget 350 --policy $policy --seed $seed --exp-name $exp_name --gpt4-mini-only fi done ================================================ FILE: abacus-research/run_mmqa_min_cost_latency.sh ================================================ #!/bin/bash for policy in "mincost" "minlatency" do for seed in {0..9} do echo "Running Seed: ${seed}" exp_name="mmqa-final-${policy}-k6-j4-budget150-seed${seed}" python mmqa-demo.py --progress --policy $policy --k 6 --j 4 --sample-budget 150 --seed $seed --exp-name $exp_name --gpt4-mini-only done done ================================================ FILE: abacus-research/score_biodex.py ================================================ import json import numpy as np def compute_final_metrics(metric: str, dir: str, exp_base_name: str): qualities = [] opt_costs, run_costs = [], [] opt_times, run_times = [], [] total_costs, total_times = [], [] print(f"--- {metric} ---") for seed in range(10): exp_name = f"{exp_base_name}-seed{seed}" with open(f"{dir}/{exp_name}-metrics.json") as f: metrics = json.load(f) qualities.append(metrics["rp@5"]) opt_costs.append(metrics["optimization_cost"]) opt_times.append(metrics["optimization_time"]) run_costs.append(metrics["plan_execution_cost"]) run_times.append(metrics["plan_execution_time"]) total_costs.append(metrics["total_execution_cost"]) total_times.append(metrics["total_execution_time"]) print(f"Opt. Cost: {np.mean(opt_costs):.3f} +/- {np.std(opt_costs):.3f}") print(f"Opt. Time: {np.mean(opt_times):.3f} +/- {np.std(opt_times):.3f}") print(f"Run Cost: {np.mean(run_costs):.3f} +/- {np.std(run_costs):.3f}") print(f"Run Time: {np.mean(run_times):.3f} +/- {np.std(run_times):.3f}") print(f"Total Cost: {np.mean(total_costs):.3f} +/- {np.std(total_costs):.3f}") print(f"Total Time: {np.mean(total_times):.3f} +/- {np.std(total_times):.3f}") print(f"Quality: {np.mean(qualities):.3f} +/- {np.std(qualities):.3f}") print("-------") if __name__ == "__main__": compute_final_metrics("quality", "opt-profiling-data", "biodex-final-mab-k6-j4-budget150") compute_final_metrics("cost", "min-cost-at-quality-data", "biodex-pareto-min-cost-budget150-k6-j4") compute_final_metrics("latency", "min-latency-at-quality-data", "biodex-pareto-min-latency-budget150-k6-j4") ================================================ FILE: abacus-research/score_cuad.py ================================================ import json import os import numpy as np def compute_final_metrics(metric: str, dir: str, exp_base_name: str): qualities = [] opt_costs, run_costs = [], [] opt_times, run_times = [], [] total_costs, total_times = [], [] print(f"--- {metric} ---") for seed in range(10): exp_name = f"{exp_base_name}-seed{seed}" if os.path.exists(f"{dir}/{exp_name}-metrics.json") is False: print(f"Missing {dir}/{exp_name}-metrics.json") continue with open(f"{dir}/{exp_name}-metrics.json") as f: metrics = json.load(f) qualities.append(metrics["f1"]) opt_costs.append(metrics["optimization_cost"]) opt_times.append(metrics["optimization_time"]) run_costs.append(metrics["plan_execution_cost"]) run_times.append(metrics["plan_execution_time"]) total_costs.append(metrics["total_execution_cost"]) total_times.append(metrics["total_execution_time"]) print(f"Opt. Cost: {np.mean(opt_costs):.3f} +/- {np.std(opt_costs):.3f}") print(f"Opt. Time: {np.mean(opt_times):.3f} +/- {np.std(opt_times):.3f}") print(f"Run Cost: {np.mean(run_costs):.3f} +/- {np.std(run_costs):.3f}") print(f"Run Time: {np.mean(run_times):.3f} +/- {np.std(run_times):.3f}") print(f"Total Cost: {np.mean(total_costs):.3f} +/- {np.std(total_costs):.3f}") print(f"Total Time: {np.mean(total_times):.3f} +/- {np.std(total_times):.3f}") print(f"Quality: {np.mean(qualities):.3f} +/- {np.std(qualities):.3f}") print("-------") if __name__ == "__main__": compute_final_metrics("quality", "opt-profiling-data", "cuad-final-mab-k6-j4-budget50") compute_final_metrics("cost", "opt-profiling-data", "cuad-final-mincost-k6-j4-budget50") compute_final_metrics("latency", "opt-profiling-data", "cuad-final-minlatency-k6-j4-budget50") ================================================ FILE: abacus-research/score_mmqa.py ================================================ import json import numpy as np def compute_final_metrics(metric: str, dir: str, exp_base_name: str): qualities = [] opt_costs, run_costs = [], [] opt_times, run_times = [], [] total_costs, total_times = [], [] print(f"--- {metric} ---") for seed in range(10): exp_name = f"{exp_base_name}-seed{seed}" with open(f"{dir}/{exp_name}-stats.json") as f: metrics = json.load(f) qualities.append(metrics["f1"]) opt_costs.append(metrics["optimization_cost"]) opt_times.append(metrics["optimization_time"]) run_costs.append(metrics["plan_execution_cost"]) run_times.append(metrics["plan_execution_time"]) total_costs.append(metrics["total_execution_cost"]) total_times.append(metrics["total_execution_time"]) print(f"Opt. Cost: {np.mean(opt_costs):.3f} +/- {np.std(opt_costs):.3f}") print(f"Opt. Time: {np.mean(opt_times):.3f} +/- {np.std(opt_times):.3f}") print(f"Run Cost: {np.mean(run_costs):.3f} +/- {np.std(run_costs):.3f}") print(f"Run Time: {np.mean(run_times):.3f} +/- {np.std(run_times):.3f}") print(f"Total Cost: {np.mean(total_costs):.3f} +/- {np.std(total_costs):.3f}") print(f"Total Time: {np.mean(total_times):.3f} +/- {np.std(total_times):.3f}") print(f"Quality: {np.mean(qualities):.3f} +/- {np.std(qualities):.3f}") print("-------") if __name__ == "__main__": compute_final_metrics("quality", "opt-profiling-data", "mmqa-final-mab-k6-j4-budget150") compute_final_metrics("cost", "opt-profiling-data", "mmqa-final-mincost-k6-j4-budget150") compute_final_metrics("latency", "opt-profiling-data", "mmqa-final-minlatency-k6-j4-budget150") ================================================ FILE: abacus-research/score_mmqa_complex.py ================================================ import json import os import numpy as np def compute_final_metrics(metric: str, dir: str, exp_base_name: str): qualities = [] opt_costs, run_costs = [], [] opt_times, run_times = [], [] total_costs, total_times = [], [] print(f"--- {metric} ---") for seed in [0, 1, 2, 3, 4, 6, 8, 9]: exp_name = f"{exp_base_name}-seed{seed}" if os.path.exists(f"{dir}/{exp_name}-stats.json"): with open(f"{dir}/{exp_name}-stats.json") as f: metrics = json.load(f) qualities.append(metrics["f1"]) opt_costs.append(metrics["optimization_cost"]) opt_times.append(metrics["optimization_time"]) run_costs.append(metrics["plan_execution_cost"]) run_times.append(metrics["plan_execution_time"]) total_costs.append(metrics["total_execution_cost"]) total_times.append(metrics["total_execution_time"]) print(f"Opt. Cost: {np.mean(opt_costs):.3f} +/- {np.std(opt_costs):.3f}") print(f"Opt. Time: {np.mean(opt_times):.3f} +/- {np.std(opt_times):.3f}") print(f"Run Cost: {np.mean(run_costs):.3f} +/- {np.std(run_costs):.3f}") print(f"Run Time: {np.mean(run_times):.3f} +/- {np.std(run_times):.3f}") print(f"Total Cost: {np.mean(total_costs):.3f} +/- {np.std(total_costs):.3f}") print(f"Total Time: {np.mean(total_times):.3f} +/- {np.std(total_times):.3f}") print(f"Quality: {np.mean(qualities):.3f} +/- {np.std(qualities):.3f}") print("-------") if __name__ == "__main__": compute_final_metrics("quality", "opt-profiling-data", "mmqa-complex-final-mab-k6-j4-budget350") compute_final_metrics("cost", "opt-profiling-data", "mmqa-complex-mincost-k6-j4-budget350") compute_final_metrics("latency", "opt-profiling-data", "mmqa-complex-minlatency-k6-j4-budget350") ================================================ FILE: abacus-research/setup_cuad_data.py ================================================ #!/usr/bin/env python """ Script to download CUAD dataset and set up local data directory. This replaces the need for HuggingFace datasets library. """ import os import urllib.request import zipfile def setup_cuad_data(): # Create cuad-data directory data_dir = "cuad-data" if not os.path.exists(data_dir): os.makedirs(data_dir) print(f"Created directory: {data_dir}") # Download CUAD data zip file data_url = "https://github.com/TheAtticusProject/cuad/raw/main/data.zip" zip_path = os.path.join(data_dir, "data.zip") if not os.path.exists(zip_path): print(f"Downloading CUAD data from {data_url}...") urllib.request.urlretrieve(data_url, zip_path) print(f"Downloaded to {zip_path}") else: print(f"Data already downloaded at {zip_path}") # Extract the zip file print("Extracting data...") with zipfile.ZipFile(zip_path, 'r') as zip_ref: zip_ref.extractall(data_dir) print(f"Extracted data to {data_dir}") # Download the dataset loading script (for reference, not actually used) script_url = "https://huggingface.co/datasets/theatticusproject/cuad-qa/resolve/main/cuad-qa.py" script_path = os.path.join(data_dir, "cuad-qa.py") if not os.path.exists(script_path): print(f"Downloading CUAD dataset script from {script_url}...") urllib.request.urlretrieve(script_url, script_path) print(f"Downloaded to {script_path}") else: print(f"Script already exists at {script_path}") # List extracted files print("\nExtracted files:") for file in os.listdir(data_dir): if file.endswith('.json'): file_path = os.path.join(data_dir, file) size = os.path.getsize(file_path) / (1024 * 1024) # Size in MB print(f" - {file} ({size:.2f} MB)") print("\nSetup complete! CUAD data is ready in the 'cuad-data' directory.") print("\nTo use this data in your scripts, update the data loading to:") print(" - train data: cuad-data/train_separate_questions.json") print(" - test data: cuad-data/test.json") return data_dir if __name__ == "__main__": setup_cuad_data() ================================================ FILE: demos/audio-demo.py ================================================ import os import kagglehub import palimpzest as pz class SmallAudioDataset(pz.AudioFileDataset): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Limit to first 10 audio files for demo purposes self.filepaths = self.filepaths[:10] if __name__ == "__main__": # Download latest version path = kagglehub.dataset_download("rushibalajiputthewad/sound-classification-of-animal-voice") print(f"Dataset downloaded to: {path}") # create simple plan to classify animal sounds plan = SmallAudioDataset(id="animal-sounds", path=os.path.join(path, "Animal-Soundprepros")) plan = plan.sem_map(cols=[{"name": "animal", "type": str, "description": "The type of animal making the sound in the recording."}]) # run plan un-optimized config = pz.QueryProcessorConfig( policy=pz.MaxQuality(), # available_models=[pz.Model.GEMINI_2_0_FLASH, pz.Model.GEMINI_2_5_FLASH, pz.Model.GEMINI_2_5_PRO], ) output = plan.run(config) print(output.to_df()) ================================================ FILE: demos/caching-demo.py ================================================ #!/usr/bin/env python3 """ Realistic Demo showcasing prompt caching capabilities in Palimpzest. This demo processes multiple employee travel requests against a comprehensive Corporate Travel Policy. The policy text (~2000 tokens) is included in the system prompt, creating a realistic scenario for prompt caching where a large static context is reused across multiple dynamic inputs. Workload: - Context: A lengthy 10-page Corporate Travel & Expense Policy. - Input: Short email requests from employees. - Task: Analyze each request for policy compliance, identifying violations and reimbursable amounts. Supported caching providers: - OpenAI (GPT-4o, GPT-4o-mini): Automatic prefix caching - Anthropic (Claude 3.5 Sonnet/Haiku): Explicit cache_control markers - Gemini: Implicit caching """ import argparse import os import time from typing import List from dotenv import load_dotenv import palimpzest as pz from palimpzest.constants import Model from palimpzest.core.lib.schemas import TextFile load_dotenv() # ============================================================================= # MOCK DATA: CORPORATE TRAVEL POLICY (Static Context > 1024 tokens) # ============================================================================= CORPORATE_TRAVEL_POLICY = """ GLOBAL CORP TRAVEL & EXPENSE POLICY (v2024.1) SECTION 1: OVERVIEW AND PHILOSOPHY Global Corp expects employees to act responsibly and professionally when incurring and submitting costs. The company will reimburse employees for reasonable and necessary expenses incurred during approved business travel. This policy applies to all employees, contractors, and consultants. SECTION 2: AIR TRAVEL 2.1 Booking Window: All domestic flights must be booked at least 14 days in advance. International flights must be booked 21 days in advance. 2.2 Class of Service: - Economy Class: Required for all domestic flights under 6 hours. - Premium Economy: Allowed for domestic flights over 6 hours or international flights under 8 hours. - Business Class: Allowed for international flights exceeding 8 hours duration. - First Class: Strictly prohibited unless approved by the CEO. 2.3 Ancillary Fees: - Checked Bags: Up to two bags reimbursed for trips > 3 days. One bag for trips <= 3 days. - Wi-Fi: Reimbursed only if business justification is provided (e.g., "urgent client deadline"). - Seat Selection: Fees > $50 require VP approval. SECTION 3: LODGING 3.1 Hotel Caps (Nightly Rates excluding taxes): - Tier 1 Cities (NY, London, Tokyo, SF, Zurich): $350 USD - Tier 2 Cities (Chicago, Paris, Berlin, Austin): $250 USD - All Other Locations: $175 USD 3.2 Room Type: Standard single rooms only. Suites are prohibited. 3.3 Laundry: Reasonable laundry expenses reimbursed for trips exceeding 5 consecutive nights. SECTION 4: MEALS AND ENTERTAINMENT 4.1 Daily Meal Allowance (Per Diem): - Tier 1 Cities: $100/day - Tier 2 Cities: $75/day - Others: $60/day 4.2 Client Entertainment: - Must include at least one current or prospective client. - Cap is $150 per person (including employees). - Names and affiliations of all attendees must be documented. 4.3 Alcohol: - Reimbursable only with dinner. - Moderate consumption allowed (max 2 drinks per person). - "Top Shelf" liquors prohibited. SECTION 5: GROUND TRANSPORTATION 5.1 Ride Share/Taxi: Preferred mode for travel between airport and hotel. 5.2 Car Rentals: - Class: Intermediate/Mid-size or smaller. - Insurance: Decline CDW/LDW (covered by corporate policy). - Fuel: Pre-paid fuel options are prohibited; cars must be returned full. 5.3 Rail: Economy/Standard class only. Acela Business Class permitted for Northeast Corridor travel. SECTION 6: MISCELLANEOUS 6.1 Tipping: - Meals: 15-20% - Taxis: 10-15% - Bellhop: $1-2 per bag 6.2 Non-Reimbursable Items: - Personal grooming/toiletries. - Fines (parking, speeding). - Airline club memberships. - In-room movies. - Lost luggage/property. SECTION 7: SUBMISSION PROCESS Expenses must be submitted within 30 days of trip completion. Receipts required for all expenses > $25. """ # ============================================================================= # MOCK DATA: EMPLOYEE REQUESTS (Dynamic Inputs) # ============================================================================= EMPLOYEE_REQUESTS = [ # Request 1: Compliant """Subject: Trip to London I booked a flight to London (8.5 hours) in Business Class for the client summit. Hotel is $320/night. Meal expenses were about $90/day. Receipts attached.""", # Request 2: Violation (Booking window & First Class) """Subject: Urgent NY Trip I need to fly to New York tomorrow. Booked First Class because it was the only seat left. Hotel is the Ritz at $500/night. Also expensed $40 for in-flight Wi-Fi to finish the Q3 report.""", # Request 3: Violation (Car Rental & Alcohol) """Subject: Austin Conference Rented a luxury SUV for the team in Austin. Dinner with the team (no clients) came to $800 ($200/person) including 3 bottles of wine. Hotel was $240/night.""", # Request 4: Compliant (Tier 2 City) """Subject: Berlin Site Visit Flew Economy to Berlin. Hotel was $220/night. Took a taxi from TXL ($45 + $5 tip). Daily meals averaged $70.""", # Request 5: Violation (Misc items) """Subject: Tokyo Tech Symposium Trip duration: 4 days. Expensed: - Flight (Premium Econ, 11 hours) - Hotel ($340/night) - Laundry service ($60) - Forgotten toothbrush replacement ($15) - Parking ticket ($50) """, ] # Output Schema OUTPUT_SCHEMA = [ {"name": "status", "type": str, "desc": "One of: 'COMPLIANT', 'PARTIAL_VIOLATION', 'MAJOR_VIOLATION'"}, { "name": "violations", "type": str, "desc": "A list of specific policy violations found, referencing the specific section numbers (e.g., 'Violation of Section 2.2'). If compliant, return 'None'.", }, { "name": "reimbursable_summary", "type": str, "desc": "A concise summary of what should be reimbursed vs rejected based on the policy text.", }, { "name": "flag_for_review", "type": bool, "desc": "True if the request requires manual review by a manager (e.g. for high amounts or ambiguous justifications).", }, ] TASK_DESC = f""" You are an AI auditor for Global Corp. Your job is to review employee travel expense descriptions against the Corporate Travel Policy. The full policy text is provided below. {CORPORATE_TRAVEL_POLICY} Analyze the input email and determine if the expenses adhere to the policy. """ class TravelRequestDataset(pz.IterDataset): """Custom dataset that provides travel requests as text records.""" def __init__(self, requests: List[str]): super().__init__(id="travel_requests", schema=TextFile) self.requests = requests def __len__(self): return len(self.requests) def __getitem__(self, idx: int): return { "filename": f"request_{idx + 1}.txt", "contents": self.requests[idx], } # Model mapping (Same as original) MODEL_MAPPING = { "gpt-4o": Model.GPT_4o, "gpt-4o-mini": Model.GPT_4o_MINI, "claude-4-0-sonnet": Model.CLAUDE_4_SONNET, # "claude-3-7-sonnet": Model.CLAUDE_3_7_SONNET, # deprecated model testing "claude-4-5-haiku": Model.CLAUDE_4_5_HAIKU, "gemini-2.5-flash": Model.GOOGLE_GEMINI_2_5_FLASH, # "deepseek-v3": Model.DEEPSEEK_V3, } def get_model_from_string(model_str: str) -> Model: if model_str.lower() in MODEL_MAPPING: return MODEL_MAPPING[model_str.lower()] for model in Model: if model.value.lower() == model_str.lower(): return model raise ValueError(f"Unknown model: {model_str}") def print_cache_stats(execution_stats): """Print cache-related statistics from execution.""" print("\n" + "=" * 60) print(" CACHE STATISTICS & COST ANALYSIS") print("=" * 60) # Token counts are now disjoint: # - input_text_tokens: regular (non-cached) input tokens # - cache_read_tokens: tokens read from cache (hits) # - cache_creation_tokens: tokens written to cache regular_input = execution_stats.input_text_tokens cache_read = execution_stats.cache_read_tokens cache_creation = execution_stats.cache_creation_tokens total_output = execution_stats.output_text_tokens total_embedding = execution_stats.embedding_input_tokens # Logical total = regular + cache read + cache creation logical_total_input = regular_input + cache_read + cache_creation print(f"{'Metric':<35} | {'Count':<15}") print("-" * 55) print(f"{'Logical Total Input Tokens':<35} | {logical_total_input:,}") print(f"{' - Regular Input (full rate)':<35} | {regular_input:,}") print(f"{' - Cache Read (discounted)':<35} | {cache_read:,}") print(f"{' - Cache Creation':<35} | {cache_creation:,}") print("-" * 55) print(f"{'Total Output Tokens':<35} | {total_output:,}") if total_embedding > 0: print(f"{'Total Embedding Input Tokens':<35} | {total_embedding:,}") print("-" * 55) print(f"{'Total Execution Cost':<35} | ${execution_stats.total_execution_cost:.6f}") # Calculate and display cache hit rate # Hit rate = cache_read / (regular_input + cache_read) total_cacheable = regular_input + cache_read if total_cacheable > 0: hit_rate = (cache_read / total_cacheable) * 100 print(f"\nCache Hit Rate: {hit_rate:.1f}%") def main(): parser = argparse.ArgumentParser(description="Demo showcasing prompt caching in Palimpzest") parser.add_argument("--model", type=str, default="gpt-4o-mini", help="Model to use") parser.add_argument("--num-records", type=int, default=5, help="Number of requests to process") parser.add_argument("--verbose", action="store_true", help="Enable verbose output") parser.add_argument("--profile", action="store_true", help="Save profiling data") args = parser.parse_args() model = get_model_from_string(args.model) # Validate env vars (Simplified for brevity) if model.is_provider_openai() and not os.getenv("OPENAI_API_KEY"): print("ERROR: OPENAI_API_KEY not set") return if model.is_provider_anthropic() and not os.getenv("ANTHROPIC_API_KEY"): print("ERROR: ANTHROPIC_API_KEY not set") return if (model.is_provider_google_ai_studio() or model.is_provider_vertex_ai()) and not os.getenv("GOOGLE_API_KEY"): print("ERROR: GOOGLE_API_KEY not set") return print("=" * 60) print(" PZ CACHING DEMO: CORPORATE AUDIT") print("=" * 60) print(f"Model: {model.value}") print( f"Policy Context Size: ~{len(CORPORATE_TRAVEL_POLICY.split())} words (~{int(len(CORPORATE_TRAVEL_POLICY.split()) * 1.3)} tokens)" ) # Repeat the request list if user wants more records than we have mocks base_requests = EMPLOYEE_REQUESTS requests = [] while len(requests) < args.num_records: requests.extend(base_requests) requests = requests[: args.num_records] print(f"Processing {len(requests)} travel requests...") # Build Plan dataset = TravelRequestDataset(requests) # The 'desc' field incorporates the huge CORPORATE_TRAVEL_POLICY string. # This ensures the System Prompt is large (>1024 tokens) and identical for all records. plan = dataset.sem_map(OUTPUT_SCHEMA, desc=TASK_DESC) config = pz.QueryProcessorConfig( policy=pz.MaxQuality(), verbose=args.verbose, execution_strategy="sequential", # Sequential often easier to debug caching behavior initially available_models=[model], ) start_time = time.time() result = plan.run(config) end_time = time.time() # Output Results print("\n" + "=" * 60) print(" AUDIT RESULTS") print("=" * 60) for i, record in enumerate(result.data_records): print(f"\n[Request {i + 1}]") print(f"Status: {record.status}") print(f"Violations: {record.violations}") print(f"Summary: {record.reimbursable_summary}") print_cache_stats(result.execution_stats) print(f"\nWall Clock Time: {end_time - start_time:.2f}s") if __name__ == "__main__": main() ================================================ FILE: demos/demo_core.py ================================================ #!/usr/bin/env python3 import json import os import pandas as pd from tabulate import tabulate import palimpzest as pz from palimpzest.core.elements.groupbysig import GroupBySig from palimpzest.core.elements.records import DataRecord sci_paper_cols = [ {"name": "title", "type": str, "desc": "The title of the paper. This is a natural language title, not a number or letter."}, {"name": "publication_year", "type": int, "desc": "The year the paper was published. This is a number."}, {"name": "author", "type": str, "desc": "The name of the first author of the paper"}, {"name": "institution", "type": str, "desc": "The institution of the first author of the paper"}, {"name": "journal", "type": str, "desc": "The name of the journal the paper was published in"}, {"name": "funding_agency", "type": str, "desc": "The name of the funding agency that supported the research"}, ] email_cols = [ {"name": "sender", "type": str, "desc": "The email address of the sender"}, {"name": "subject", "type": str, "desc": "The subject of the email"}, ] dog_image_cols = [ {"name": "breed", "type": str, "desc": "The breed of the dog"}, ] def build_sci_paper_plan(dataset): """A dataset-independent declarative description of authors of good papers""" return pz.PDFFileDataset(id="science-papers", path=dataset).sem_map(sci_paper_cols) def build_test_pdf_plan(dataset): """This tests whether we can process a PDF file""" return pz.PDFFileDataset(id="pdf-files", path=dataset) def build_mit_battery_paper_plan(dataset): """A dataset-independent declarative description of authors of good papers""" sci_papers = pz.PDFFileDataset(id="science-papers", path=dataset).sem_map(sci_paper_cols) battery_papers = sci_papers.sem_filter("The paper is about batteries") mit_papers = battery_papers.sem_filter("The paper is from MIT") return mit_papers def build_enron_plan(dataset): """Build a plan for processing Enron email data""" return pz.TextFileDataset(id="enron-emails", path=dataset).sem_map(email_cols) def compute_enron_stats(dataset): """Compute statistics on Enron email data""" emails = pz.TextFileDataset(id="enron-emails", path=dataset).sem_map(email_cols) subject_line_lengths = emails.sem_map([{"name": "words", "type": int, "desc": "The number of words in the subject field"}]) return subject_line_lengths def enron_gby_plan(dataset): """Group Enron emails by sender""" emails = pz.TextFileDataset(id="enron-emails", path=dataset).sem_map(email_cols) ops = ["count"] fields = ["sender"] groupbyfields = ["sender"] gby_desc = GroupBySig(groupbyfields, ops, fields) grouped_emails = emails.groupby(gby_desc) return grouped_emails def enron_count_plan(dataset): """Count total Enron emails""" emails = pz.TextFileDataset(id="enron-emails", path=dataset).sem_map(email_cols) ops = ["count"] fields = ["sender"] groupbyfields = [] gby_desc = GroupBySig(groupbyfields, ops, fields) count_emails = emails.groupby(gby_desc) return count_emails def enron_average_count_plan(dataset): """Calculate average number of emails per sender""" emails = pz.TextFileDataset(id="enron-emails", path=dataset).sem_map(email_cols) ops = ["count"] fields = ["sender"] groupbyfields = ["sender"] gby_desc = GroupBySig(groupbyfields, ops, fields) grouped_emails = emails.groupby(gby_desc) ops = ["average"] fields = ["count(sender)"] groupbyfields = [] gby_desc = GroupBySig(groupbyfields, ops, fields) average_emails_per_sender = grouped_emails.groupby(gby_desc) return average_emails_per_sender def enron_limit_plan(dataset, limit=5): """Get limited number of Enron emails""" emails = pz.TextFileDataset(id="enron-emails", path=dataset).sem_map(email_cols) limit_data = emails.limit(limit) return limit_data def build_image_plan(dataset): """Build a plan for processing dog images""" images = pz.ImageFileDataset(id="dog-images", path=dataset) filtered_images = images.sem_filter("The image contains one or more dogs") dog_images = filtered_images.sem_map(dog_image_cols) return dog_images def build_image_agg_plan(dataset): """Build a plan for aggregating dog images by breed""" images = pz.ImageFileDataset(id="dog-images", path=dataset) filtered_images = images.sem_filter("The image contains one or more dogs") dog_images = filtered_images.sem_map(dog_image_cols) ops = ["count"] fields = ["breed"] groupbyfields = ["breed"] gby_desc = GroupBySig(groupbyfields, ops, fields) grouped_dog_images = dog_images.groupby(gby_desc) return grouped_dog_images def build_join_plan(dataset1, dataset2): """Build a plan that joins two datasets""" ds1 = pz.TextFileDataset(id="enron-emails", path=dataset1).sem_map(email_cols) ds2 = pz.TextFileDataset(id="other-enron-emails", path=dataset2).sem_map(email_cols) joined = ds1.sem_join(ds2, condition="sender") return joined def build_join_image_plan(dataset1, dataset2): """Build a plan that joins two datasets with images""" ds1 = pz.ImageFileDataset(id="dog-images", path=dataset1).sem_map(dog_image_cols) ds2 = pz.ImageFileDataset(id="other-dog-images", path=dataset2).sem_map(dog_image_cols) joined = ds1.sem_join(ds2, condition="breed") return joined def get_task_config(task, dataset, join_dataset=None): """Get configuration for a specific task""" if task == "paper": root_set = build_mit_battery_paper_plan(dataset) cols = ["title", "publication_year", "author", "institution", "journal", "funding_agency"] stat_path = "profiling-data/paper-profiling.json" elif task == "enron": root_set = build_enron_plan(dataset) cols = ["sender", "subject"] stat_path = "profiling-data/enron-profiling.json" elif task == "enronGby": root_set = enron_gby_plan(dataset) cols = ["sender", "count(sender)"] stat_path = "profiling-data/egby-profiling.json" elif task in ("enronCount", "count"): root_set = enron_count_plan(dataset) cols = ["count(sender)"] stat_path = "profiling-data/ecount-profiling.json" elif task in ("enronAvgCount", "average"): root_set = enron_average_count_plan(dataset) cols = ["average(count(sender))"] stat_path = "profiling-data/e-profiling.json" elif task == "enronmap": root_set = compute_enron_stats(dataset) cols = ["sender", "subject", "value"] stat_path = "profiling-data/emap-profiling.json" elif task == "pdftest": root_set = build_test_pdf_plan(dataset) cols = ["filename"] stat_path = "profiling-data/pdftest-profiling.json" elif task == "scitest": root_set = build_sci_paper_plan(dataset) cols = ["title", "author", "institution", "journal", "funding_agency"] stat_path = "profiling-data/scitest-profiling.json" elif task == "image": root_set = build_image_plan(dataset) cols = None stat_path = "profiling-data/image-profiling.json" elif task == "gbyImage": root_set = build_image_agg_plan(dataset) cols = ["breed", "count(breed)"] stat_path = "profiling-data/gbyImage-profiling.json" elif task == "limit": root_set = enron_limit_plan(dataset, 5) cols = ["sender", "subject"] stat_path = "profiling-data/limit-profiling.json" elif task == "join": root_set = build_join_plan(dataset, join_dataset) cols = ["filename", "sender", "subject"] stat_path = "profiling-data/join-profiling.json" elif task == "joinImage": root_set = build_join_image_plan(dataset, join_dataset) cols = None stat_path = "profiling-data/joinImage-profiling.json" else: raise ValueError(f"Unknown task: {task}") return root_set, cols, stat_path def execute_task(task, dataset, policy, join_dataset=None, verbose=False, profile=False, execution_strategy="sequential", optimizer_strategy="pareto"): """Execute a task and return results""" root_set, cols, stat_path = get_task_config(task, dataset, join_dataset) config = pz.QueryProcessorConfig( policy=policy, verbose=verbose, execution_strategy=execution_strategy, optimizer_strategy=optimizer_strategy, ) data_record_collection = root_set.run(config) if profile: os.makedirs("profiling-data", exist_ok=True) with open(stat_path, "w") as f: json.dump(data_record_collection.execution_stats.to_json(), f) return data_record_collection.data_records, data_record_collection.execution_stats, cols def format_results_table(records: list[DataRecord], cols=None): """Format records as a table""" records = [record.to_dict(include_bytes=False) for record in records] records_df = pd.DataFrame(records) print_cols = records_df.columns if cols is None else cols final_df = records_df[print_cols] if not records_df.empty else pd.DataFrame(columns=print_cols) return tabulate(final_df, headers="keys", tablefmt="psql") ================================================ FILE: demos/enron-demo.py ================================================ import json import os import palimpzest as pz from palimpzest.core.lib.schemas import TextFile class EnronValidator(pz.Validator): def __init__(self, labels_file: str): super().__init__() self.filename_to_labels = {} if labels_file: with open(labels_file) as f: self.filename_to_labels = json.load(f) def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: filename = input_record["filename"] labels = self.filename_to_labels[filename] if len(labels) == 0: return None labels = labels[0] return (float(labels["sender"] == output["sender"]) + float(labels["subject"] == output["subject"])) / 2.0 class EnronDataset(pz.IterDataset): def __init__(self, dir: str, labels_file: str | None = None, split: str = "test"): super().__init__(id="enron", schema=TextFile) self.filepaths = [os.path.join(dir, filename) for filename in os.listdir(dir)] self.filepaths = self.filepaths[:50] if split == "train" else self.filepaths[50:150] self.filename_to_labels = {} if labels_file: with open(labels_file) as f: self.filename_to_labels = json.load(f) def __len__(self): return len(self.filepaths) def __getitem__(self, idx: int): # get input fields filepath = self.filepaths[idx] filename = os.path.basename(filepath) with open(filepath) as f: contents = f.read() # create item with fields item = {"filename": filename, "contents": contents} return item if __name__ == "__main__": # create validator and train_dataset validator = EnronValidator(labels_file="testdata/enron-eval-medium-labels.json") train_dataset = EnronDataset(dir="testdata/enron-eval-medium", split="train") # construct plan plan = EnronDataset(dir="testdata/enron-eval-medium", split="test") plan = plan.sem_map([ {"name": "subject", "type": str, "desc": "The subject of the email"}, {"name": "sender", "type": str, "desc": "The email address of the email's sender"}, ]) plan = plan.sem_filter( 'The email refers to a fraudulent scheme (i.e., "Raptor", "Deathstar", "Chewco", and/or "Fat Boy")', depends_on=["contents"], ) plan = plan.sem_filter( "The email is not quoting from a news article or an article written by someone outside of Enron", depends_on=["contents"], ) # execute pz plan config = pz.QueryProcessorConfig( policy=pz.MaxQuality(), execution_strategy="parallel", k=5, j=6, sample_budget=100, max_workers=20, progress=True, ) output = plan.optimize_and_run(train_dataset=train_dataset, validator=validator, config=config) # print output dataframe print(output.to_df()) # print precision and recall with open("testdata/enron-eval-medium-labels.json") as f: filename_to_labels = json.load(f) test_filenames = os.listdir("testdata/enron-eval-medium")[50:150] filename_to_labels = {k: v for k, v in filename_to_labels.items() if k in test_filenames} target_filenames = set(filename for filename, labels in filename_to_labels.items() if labels != []) pred_filenames = set(output.to_df()["filename"]) tp = sum(filename in target_filenames for filename in pred_filenames) fp = len(pred_filenames) - tp fn = len(target_filenames) - tp print(f"PRECISION: {tp/(tp + fp) if tp + fp > 0 else 0.0:.3f}") print(f"RECALL: {tp/(tp + fn) if tp + fn > 0 else 0.0:.3f}") ================================================ FILE: demos/image-demo.py ================================================ #!/usr/bin/env python3 """This scripts is a demo for image processing, it is simply an abridged version of simpleDemo.py""" import os import time import gradio as gr import numpy as np from PIL import Image import palimpzest as pz if not os.environ.get("OPENAI_API_KEY"): from palimpzest.utils.env_helpers import load_env load_env() dog_image_cols = [ {"name": "breed", "type": str, "desc": "The breed of the dog"}, ] def build_image_plan(dataset): images = pz.ImageFileDataset(id="images", path=dataset) filtered_images = images.sem_filter("The image contains one or more dogs") dog_images = filtered_images.sem_map(dog_image_cols) return dog_images if __name__ == "__main__": # parse arguments start_time = time.time() if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") print("Starting image task") policy = pz.MaxQuality() plan = build_image_plan("testdata/images-tiny") config = pz.QueryProcessorConfig(policy=policy) data_record_collection = plan.run(config) imgs, breeds = [], [] for record in data_record_collection: print("Trying to open ", record.filename) path = os.path.join("testdata/images-tiny/", record.filename) img = Image.open(path).resize((128, 128)) img_arr = np.asarray(img) imgs.append(img_arr) breeds.append(record.breed) with gr.Blocks() as demo: img_blocks, breed_blocks = [], [] for img, breed in zip(imgs, breeds): with gr.Row(): with gr.Column(): img_blocks.append(gr.Image(value=img)) with gr.Column(): breed_blocks.append(gr.Textbox(value=breed)) plan_str = list(data_record_collection.execution_stats.plan_strs.values())[0] gr.Textbox(value=plan_str, info="Query Plan") end_time = time.time() print("Elapsed time:", end_time - start_time) demo.launch() ================================================ FILE: demos/join-data/animal-texts/animal1.txt ================================================ The quick red fox jumped over the fence. ================================================ FILE: demos/join-data/animal-texts/animal2.txt ================================================ The black dog sat next to the bed. ================================================ FILE: demos/join-data/animal-texts/animal3.txt ================================================ The white polar bear swam gently in the ocean. ================================================ FILE: demos/join-data/animal-texts/animal4.txt ================================================ The labrador swam in the lake, the sun glistening off its shiny black coat. ================================================ FILE: demos/join-data/animal-texts/animal5.txt ================================================ Clifford was a big red dog. ================================================ FILE: demos/join-data/animal-texts/animal6.txt ================================================ The elephant was wise and grey. ================================================ FILE: demos/join-demo.py ================================================ import argparse import palimpzest as pz # define columns for datasets text_animal_cols = [ {"name": "animal", "type": str, "desc": "The type of animal mentioned in the text"}, {"name": "color", "type": str, "desc": "The color of the animal mentioned in the text"}, ] image_animal_cols = [ {"name": "animal", "type": str, "desc": "The type of animal in the image"}, {"name": "color", "type": str, "desc": "The color of the animal in the image"}, ] # query plans def run_text_join(): """Build a plan that joins two datasets""" ds1 = pz.TextFileDataset(id="animals1", path="join-data/animal-texts/").sem_map(text_animal_cols) ds2 = pz.TextFileDataset(id="animals2", path="join-data/animal-texts/").sem_map(text_animal_cols) ds3 = ds1.sem_join(ds2, condition="both animals are canines with the same color") config = pz.QueryProcessorConfig( policy=pz.MaxQuality(), execution_strategy="parallel", join_parallelism=64, ) data_record_collection = ds3.run(config) print(data_record_collection.to_df()) def run_image_join(): """Build a plan that joins two datasets with images""" ds1 = pz.ImageFileDataset(id="animals1", path="join-data/animal-images/").sem_map(image_animal_cols) ds2 = pz.ImageFileDataset(id="animals2", path="join-data/animal-images/").sem_map(image_animal_cols) ds3 = ds1.sem_join(ds2, condition="both animals are canines with the same color") config = pz.QueryProcessorConfig( policy=pz.MaxQuality(), execution_strategy="parallel", join_parallelism=64, ) data_record_collection = ds3.run(config) print(data_record_collection.to_df()) def run_text_image_join(): """Build a plan that joins a dataset with text to a dataset with images""" ds1 = pz.TextFileDataset(id="animals1", path="join-data/animal-texts/").sem_map(text_animal_cols) ds2 = pz.ImageFileDataset(id="animals2", path="join-data/animal-images/").sem_map(image_animal_cols) ds3 = ds1.sem_join(ds2, condition="both animals are canines with the same color") config = pz.QueryProcessorConfig( policy=pz.MaxQuality(), execution_strategy="parallel", join_parallelism=64, ) data_record_collection = ds3.run(config) print(data_record_collection.to_df()) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Run the Palimpzest join demo.") parser.add_argument("--task", type=str, help="Which join demo to run") args = parser.parse_args() if args.task == "text-join": run_text_join() elif args.task == "image-join": run_image_join() elif args.task == "text-image-join": run_text_image_join() else: print("Please provide a valid task: one of 'text-join', 'image-join', 'text-image-join'") exit(1) ================================================ FILE: demos/paper-demo.py ================================================ import argparse import json import os import gradio as gr import numpy as np import pandas as pd from PIL import Image import palimpzest as pz from palimpzest.core.lib.schemas import ImageFilepath from palimpzest.utils.udfs import xls_to_tables def print_table(records, cols=None, plan_str=None): """Helper function to print execution results using Gradio""" if len(records) == 0: print("No records met search criteria") return records = [record.to_dict() for record in records] records_df = pd.DataFrame(records) print_cols = records_df.columns if cols is None else cols with gr.Blocks() as demo: gr.Dataframe(records_df[print_cols]) if plan_str is not None: gr.Textbox(value=plan_str, info="Physical Plan") demo.launch() # Addresses far from MIT; we use a simple lookup like this to make the # experiments re-producible w/out needed a Google API key for geocoding lookups FAR_AWAY_ADDRS = [ "Melcher St", "Sleeper St", "437 D St", "Seaport Blvd", "50 Liberty Dr", "Telegraph St", "Columbia Rd", "E 6th St", "E 7th St", "E 5th St", ] def within_two_miles_of_mit(record: dict): # NOTE: I'm using this hard-coded function so that folks w/out a # Geocoding API key from google can still run this example try: return not any([street.lower() in record["address"].lower() for street in FAR_AWAY_ADDRS]) except Exception: return False def in_price_range(record: dict): try: price = record["price"] if isinstance(price, str): price = price.strip() price = int(price.replace("$", "").replace(",", "")) return 6e5 < price <= 2e6 except Exception: return False email_cols = [ {"name": "sender", "type": str, "desc": "The email address of the sender"}, {"name": "subject", "type": str, "desc": "The subject of the email"}, ] case_data_cols = [ {"name": "case_submitter_id", "type": str, "desc": "The ID of the case"}, {"name": "age_at_diagnosis", "type": int | float, "desc": "The age of the patient at the time of diagnosis"}, {"name": "race", "type": str, "desc": "An arbitrary classification of a taxonomic group that is a division of a species."}, {"name": "ethnicity", "type": str, "desc": "Whether an individual describes themselves as Hispanic or Latino or not."}, {"name": "gender", "type": str, "desc": "Text designations that identify gender."}, {"name": "vital_status", "type": str, "desc": "The vital status of the patient"}, {"name": "ajcc_pathologic_t", "type": str, "desc": "Code of pathological T (primary tumor) to define the size or contiguous extension of the primary tumor (T), using staging criteria from the American Joint Committee on Cancer (AJCC)."}, {"name": "ajcc_pathologic_n", "type": str, "desc": "The codes that represent the stage of cancer based on the nodes present (N stage) according to criteria based on multiple editions of the AJCC's Cancer Staging Manual."}, {"name": "ajcc_pathologic_stage", "type": str, "desc": "The extent of a cancer, especially whether the disease has spread from the original site to other parts of the body based on AJCC staging criteria."}, {"name": "tumor_grade", "type": int | float, "desc": "Numeric value to express the degree of abnormality of cancer cells, a measure of differentiation and aggressiveness."}, {"name": "tumor_focality", "type": str, "desc": "The text term used to describe whether the patient's disease originated in a single location or multiple locations."}, {"name": "tumor_largest_dimension_diameter", "type": int | float, "desc": "The tumor largest dimension diameter."}, {"name": "primary_diagnosis", "type": str, "desc": "Text term used to describe the patient's histologic diagnosis, as described by the World Health Organization's (WHO) International Classification of Diseases for Oncology (ICD-O)."}, {"name": "morphology", "type": str, "desc": "The Morphological code of the tumor, as described by the World Health Organization's (WHO) International Classification of Diseases for Oncology (ICD-O)."}, {"name": "tissue_or_organ_of_origin", "type": str, "desc": "The text term used to describe the anatomic site of origin, of the patient's malignant disease, as described by the World Health Organization's (WHO) International Classification of Diseases for Oncology (ICD-O)."}, {"name": "study", "type": str, "desc": "The last name of the author of the study, from the table name"}, {"name": "filename", "type": str, "desc": "The name of the file the record was extracted from"} ] real_estate_listing_cols = [ {"name": "listing", "type": str, "desc": "The name of the listing"}, {"name": "text_content", "type": str, "desc": "The content of the listing's text description"}, {"name": "image_filepaths", "type": list[ImageFilepath], "desc": "A list of the filepaths for each image of the listing"}, ] real_estate_text_cols = [ {"name": "address", "type": str, "desc": "The address of the property"}, {"name": "price", "type": int | float, "desc": "The listed price of the property"}, ] real_estate_image_cols = [ {"name": "is_modern_and_attractive", "type": bool, "desc": "True if the home interior design is modern and attractive and False otherwise"}, {"name": "has_natural_sunlight", "type": bool, "desc": "True if the home interior has lots of natural sunlight and False otherwise"}, ] table_cols = [ {"name": "rows", "type": list[str], "desc": "The rows of the table"}, {"name": "header", "type": list[str], "desc": "The header of the table"}, {"name": "name", "type": str, "desc": "The name of the table"}, {"name": "filename", "type": str, "desc": "The name of the file the table was extracted from"} ] class RealEstateListingDataset(pz.IterDataset): def __init__(self, listings_dir): super().__init__(id="real-estate", schema=real_estate_listing_cols) self.listings_dir = listings_dir self.listings = sorted(os.listdir(self.listings_dir)) self.listings = [file for file in self.listings if not file.startswith(".")] def __len__(self): return len(self.listings) def __getitem__(self, idx: int): # get listing listing = self.listings[idx] # get fields image_filepaths, text_content = [], None listing_dir = os.path.join(self.listings_dir, listing) for file in os.listdir(listing_dir): if file.endswith(".txt"): with open(os.path.join(listing_dir, file), "rb") as f: text_content = f.read().decode("utf-8") elif file.endswith(".png"): image_filepaths.append(os.path.join(listing_dir, file)) # construct and return dictionary with fields return {"listing": listing, "text_content": text_content, "image_filepaths": image_filepaths} if __name__ == "__main__": # parse arguments parser = argparse.ArgumentParser(description="Run a simple demo") parser.add_argument("--viz", default=False, action="store_true", help="Visualize output in Gradio") parser.add_argument("--verbose", default=False, action="store_true", help="Print verbose output") parser.add_argument("--profile", default=False, action="store_true", help="Profile execution") parser.add_argument("--dataset", type=str, help="The path to the dataset") parser.add_argument( "--workload", type=str, help="The workload to run. One of enron, real-estate, medical-schema-matching." ) parser.add_argument( "--executor", type=str, help="The plan executor to use. One of sequential, pipelined, parallel", default="parallel", ) parser.add_argument( "--policy", type=str, help="One of 'mincost', 'mintime', 'maxquality'", default="maxquality", ) args = parser.parse_args() # The user has to indicate the dataset id and the workload if args.dataset is None: print("Please provide a dataset id") exit(1) if args.workload is None: print("Please provide a workload") exit(1) # create directory for profiling data if args.profile: os.makedirs("profiling-data", exist_ok=True) dataset = args.dataset workload = args.workload visualize = args.viz verbose = args.verbose profile = args.profile policy = pz.MaxQuality() if args.policy == "mincost": policy = pz.MinCost() elif args.policy == "mintime": policy = pz.MinTime() elif args.policy == "maxquality": policy = pz.MaxQuality() else: print("Policy not supported for this demo") exit(1) if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") # create pz plan if workload == "enron": plan = pz.TextFileDataset(id="enron", path=dataset) plan = plan.sem_map(email_cols) plan = plan.sem_filter( "The email is not quoting from a news article or an article written by someone outside of Enron", depends_on=["contents"], ) plan = plan.sem_filter( 'The email refers to a fraudulent scheme (i.e., "Raptor", "Deathstar", "Chewco", and/or "Fat Boy")', depends_on=["contents"], ) elif workload == "real-estate": plan = RealEstateListingDataset(dataset) plan = plan.sem_map(real_estate_text_cols, depends_on="text_content") plan = plan.sem_map(real_estate_image_cols, depends_on="image_filepaths") plan = plan.sem_filter( "The interior is modern and attractive, and has lots of natural sunlight", depends_on=["is_modern_and_attractive", "has_natural_sunlight"], ) plan = plan.filter(within_two_miles_of_mit, depends_on="address") plan = plan.filter(in_price_range, depends_on="price") elif workload == "medical-schema-matching": plan = dataset.add_columns(xls_to_tables, cols=table_cols, cardinality=pz.Cardinality.ONE_TO_MANY) plan = plan.sem_filter("The rows of the table contain the patient age") plan = plan.sem_map(case_data_cols, cardinality=pz.Cardinality.ONE_TO_MANY) # construct config and run plan config = pz.QueryProcessorConfig( verbose=verbose, policy=policy, execution_strategy=args.executor, ) data_record_collection = plan.run(config) print(data_record_collection.to_df()) # save statistics if profile: stats_path = f"profiling-data/{workload}-profiling.json" execution_stats_dict = data_record_collection.execution_stats.to_json() with open(stats_path, "w") as f: json.dump(execution_stats_dict, f) # visualize output in Gradio if visualize: plan_str = list(data_record_collection.execution_stats.plan_strs.values())[-1] if workload == "enron": print_table(data_record_collection.data_records, cols=["sender", "subject"], plan_str=plan_str) elif workload == "real-estate": fst_imgs, snd_imgs, thrd_imgs, addrs, prices = [], [], [], [], [] for record in data_record_collection: addrs.append(record.address) prices.append(record.price) for idx, img_name in enumerate(["img1.png", "img2.png", "img3.png"]): path = os.path.join(dataset, record.listing, img_name) img = Image.open(path) img_arr = np.asarray(img) if idx == 0: fst_imgs.append(img_arr) elif idx == 1: snd_imgs.append(img_arr) elif idx == 2: thrd_imgs.append(img_arr) with gr.Blocks() as demo: fst_img_blocks, snd_img_blocks, thrd_img_blocks, addr_blocks, price_blocks = [], [], [], [], [] for fst_img, snd_img, thrd_img, addr, price in zip(fst_imgs, snd_imgs, thrd_imgs, addrs, prices): with gr.Row(equal_height=True): with gr.Column(): fst_img_blocks.append(gr.Image(value=fst_img)) with gr.Column(): snd_img_blocks.append(gr.Image(value=snd_img)) with gr.Column(): thrd_img_blocks.append(gr.Image(value=thrd_img)) with gr.Row(): with gr.Column(): addr_blocks.append(gr.Textbox(value=addr, info="Address")) with gr.Column(): price_blocks.append(gr.Textbox(value=price, info="Price")) plan_str = list(data_record_collection.execution_stats.plan_strs.values())[0] gr.Textbox(value=plan_str, info="Query Plan") demo.launch() ================================================ FILE: demos/real-estate-demo.py ================================================ import argparse import os import gradio as gr import numpy as np import pandas as pd from PIL import Image import palimpzest as pz from palimpzest.core.lib.schemas import ImageFilepath def print_table(records, cols=None, plan_str=None): """Helper function to print execution results using Gradio""" if len(records) == 0: print("No records met search criteria") return records = [record.to_dict() for record in records] records_df = pd.DataFrame(records) print_cols = records_df.columns if cols is None else cols with gr.Blocks() as demo: gr.Dataframe(records_df[print_cols]) if plan_str is not None: gr.Textbox(value=plan_str, info="Physical Plan") demo.launch() # Addresses far from MIT; we use a simple lookup like this to make the # experiments re-producible w/out needed a Google API key for geocoding lookups FAR_AWAY_ADDRS = [ "Melcher St", "Sleeper St", "437 D St", "Seaport Blvd", "50 Liberty Dr", "Telegraph St", "Columbia Rd", "E 6th St", "E 7th St", "E 5th St", ] def within_two_miles_of_mit(record: dict): # NOTE: I'm using this hard-coded function so that folks w/out a # Geocoding API key from google can still run this example try: return not any([street.lower() in record["address"].lower() for street in FAR_AWAY_ADDRS]) except Exception: return False def in_price_range(record: dict): try: price = record["price"] if isinstance(price, str): price = price.strip() price = int(price.replace("$", "").replace(",", "")) return 6e5 < price <= 2e6 except Exception: return False real_estate_listing_cols = [ {"name": "listing", "type": str, "desc": "The name of the listing"}, {"name": "text_content", "type": str, "desc": "The content of the listing's text description"}, {"name": "image_filepaths", "type": list[ImageFilepath], "desc": "A list of the filepaths for each image of the listing"}, ] real_estate_text_cols = [ {"name": "address", "type": str, "desc": "The address of the property"}, {"name": "price", "type": int | float, "desc": "The listed price of the property"}, ] real_estate_image_cols = [ {"name": "is_modern_and_attractive", "type": bool, "desc": "True if the home interior design is modern and attractive and False otherwise"}, {"name": "has_natural_sunlight", "type": bool, "desc": "True if the home interior has lots of natural sunlight and False otherwise"}, ] # class RealEstateValidator(pz.Validator): # def __init__(self, labels_file: str): # super().__init__() # with open(labels_file) as f: # self.filename_to_labels = json.load(f) # def filter_score_fn(self, filter_str: str, input_record: dict, output: bool) -> float | None: # filename = input_record["filename"] # labels = self.filename_to_labels[filename] # if labels is None: # return None # if "business transactions" in filter_str: # return float(labels["mentions_transaction"] == output) # elif "first-hand discussion" in filter_str: # return float(labels["firsthand_discussion"] == output) # else: # return None # def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None: # # NOTE: we score the map based on the sender and subject fields only, as summary is too subjective; # # we could also use an LLM judge within this function to score the summary field if desired # filename = input_record["filename"] # labels = self.filename_to_labels[filename] # if labels is None: # return None # return (float(labels["sender"] == output["sender"]) + float(labels["subject"] == output["subject"])) / 2.0 class RealEstateListingDataset(pz.IterDataset): def __init__(self, listings_dir): super().__init__(id="real-estate", schema=real_estate_listing_cols) self.listings_dir = listings_dir self.listings = sorted(os.listdir(self.listings_dir)) self.listings = [file for file in self.listings if not file.startswith(".")] def __len__(self): return len(self.listings) def __getitem__(self, idx: int): # get listing listing = self.listings[idx] # get fields image_filepaths, text_content = [], None listing_dir = os.path.join(self.listings_dir, listing) for file in os.listdir(listing_dir): if file.endswith(".txt"): with open(os.path.join(listing_dir, file), "rb") as f: text_content = f.read().decode("utf-8") elif file.endswith(".png"): image_filepaths.append(os.path.join(listing_dir, file)) # construct and return dictionary with fields return {"listing": listing, "text_content": text_content, "image_filepaths": image_filepaths} if __name__ == "__main__": # parse arguments parser = argparse.ArgumentParser(description="Run a simple demo") parser.add_argument("--viz", default=False, action="store_true", help="Visualize output in Gradio") parser.add_argument("--dataset", type=str, help="The path to the dataset") parser.add_argument( "--policy", type=str, help="One of 'mincost', 'mintime', 'maxquality'", default="maxquality", ) args = parser.parse_args() # The user has to indicate the dataset id and the workload if args.dataset is None: print("Please provide a dataset id") exit(1) dataset = args.dataset visualize = args.viz policy = pz.MaxQuality() if args.policy == "mincost": policy = pz.MinCost() elif args.policy == "mintime": policy = pz.MinTime() elif args.policy == "maxquality": policy = pz.MaxQuality() else: print("Policy not supported for this demo") exit(1) if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") # create pz plan plan = RealEstateListingDataset(dataset) plan = plan.sem_map(real_estate_text_cols, depends_on="text_content") plan = plan.sem_map(real_estate_image_cols, depends_on="image_filepaths") plan = plan.sem_filter( "The interior is modern and attractive, and has lots of natural sunlight", depends_on=["is_modern_and_attractive", "has_natural_sunlight"], ) plan = plan.filter(within_two_miles_of_mit, depends_on="address") plan = plan.filter(in_price_range, depends_on="price") # construct config and run plan config = pz.QueryProcessorConfig( policy=policy, available_models=[pz.Model.GPT_5_MINI], k=6, j=6, sample_budget=125, ) data_record_collection = plan.optimize_and_run(config, validator=pz.Validator(model=pz.Model.o4_MINI)) print(data_record_collection.to_df()) # preds = data_record_collection.to_df()["listing"].tolist() # gt_df = pd.read_csv("testdata/groundtruth/real-estate-eval-100.csv") # labels = gt_df.listing.tolist() # tp, fp, fn = 0, 0, 0 # for pred in preds: # if pred in labels: # tp += 1 # else: # fp += 1 # for label in labels: # if label not in preds: # fn += 1 # precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0 # recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0 # f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0 # print(f"Precision: {precision:.2f}") # print(f"Recall: {recall:.4f}") # print(f"F1: {f1:.4f}") # visualize output in Gradio if visualize: plan_str = list(data_record_collection.execution_stats.plan_strs.values())[-1] fst_imgs, snd_imgs, thrd_imgs, addrs, prices = [], [], [], [], [] for record in data_record_collection: addrs.append(record.address) prices.append(record.price) for idx, img_name in enumerate(["img1.png", "img2.png", "img3.png"]): path = os.path.join(dataset, record.listing, img_name) img = Image.open(path) img_arr = np.asarray(img) if idx == 0: fst_imgs.append(img_arr) elif idx == 1: snd_imgs.append(img_arr) elif idx == 2: thrd_imgs.append(img_arr) with gr.Blocks() as demo: fst_img_blocks, snd_img_blocks, thrd_img_blocks, addr_blocks, price_blocks = [], [], [], [], [] for fst_img, snd_img, thrd_img, addr, price in zip(fst_imgs, snd_imgs, thrd_imgs, addrs, prices): with gr.Row(equal_height=True): with gr.Column(): fst_img_blocks.append(gr.Image(value=fst_img)) with gr.Column(): snd_img_blocks.append(gr.Image(value=snd_img)) with gr.Column(): thrd_img_blocks.append(gr.Image(value=thrd_img)) with gr.Row(): with gr.Column(): addr_blocks.append(gr.Textbox(value=addr, info="Address")) with gr.Column(): price_blocks.append(gr.Textbox(value=price, info="Price")) plan_str = list(data_record_collection.execution_stats.plan_strs.values())[0] gr.Textbox(value=plan_str, info="Query Plan") demo.launch() ================================================ FILE: demos/simple-demo.py ================================================ #!/usr/bin/env python3 import argparse import os import time from demo_core import execute_task, format_results_table from dotenv import load_dotenv import palimpzest as pz load_dotenv() def main(): # parse arguments start_time = time.time() parser = argparse.ArgumentParser(description="Run a simple demo") parser.add_argument("--verbose", default=False, action="store_true", help="Print verbose output") parser.add_argument("--profile", default=False, action="store_true", help="Profile execution") parser.add_argument("--dataset", type=str, help="Path to the dataset") parser.add_argument("--join-dataset", type=str, help="Path to the join dataset (if needed)", default=None) parser.add_argument("--task", type=str, help="The task to run") parser.add_argument( "--execution-strategy", type=str, help="The execution strategy to use. One of sequential, pipelined, parallel", default="sequential", ) parser.add_argument( "--policy", type=str, help="One of 'mincost', 'mintime', 'maxquality'", default="maxquality", ) args = parser.parse_args() # The user has to indicate the dataset and the task if args.dataset is None: print("Please provide a path to the dataset") exit(1) if args.task is None: print("Please provide a task") exit(1) # Set up execution parameters dataset = args.dataset join_dataset = args.join_dataset task = args.task verbose = args.verbose profile = args.profile # Set policy policy = pz.MaxQuality() if args.policy == "mincost": policy = pz.MinCost() elif args.policy == "mintime": policy = pz.MinTime() elif args.policy == "maxquality": policy = pz.MaxQuality() else: print("Policy not supported for this demo") exit(1) if os.getenv("OPENAI_API_KEY") is None and os.getenv("TOGETHER_API_KEY") is None and os.getenv("ANTHROPIC_API_KEY") is None: print("WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset") # Execute task records, execution_stats, cols = execute_task( task=task, dataset=dataset, policy=policy, join_dataset=join_dataset, verbose=verbose, profile=profile, execution_strategy=args.execution_strategy ) # Print results print(f"Policy is: {str(policy)}") print("Executed plan:") plan_str = list(execution_stats.plan_strs.values())[0] print(plan_str) end_time = time.time() print("Elapsed time:", end_time - start_time) print(format_results_table(records, cols=cols)) if __name__ == "__main__": main() ================================================ FILE: demos/vllm-demo.py ================================================ #!/usr/bin/env python3 """ Minimal demo for running a vLLM model with Palimpzest. Prerequisites: 1. Start a vLLM server serving a small model, e.g.: vllm serve Qwen/Qwen2.5-1.5B-Instruct --port 8000 2. Run this script: python demos/vllm-demo.py \ --api-base http://localhost:8000/v1 \ --model-id openai/Qwen/Qwen2.5-1.5B-Instruct """ import argparse import os from pydantic import BaseModel, Field import palimpzest as pz class SentimentResult(BaseModel): sentiment: str = Field(description="The sentiment of the text: positive, negative, or neutral") def main(): parser = argparse.ArgumentParser(description="Run a minimal vLLM demo") parser.add_argument("--api-base", type=str, required=True, help="vLLM server base URL (e.g. http://localhost:8000/v1)") parser.add_argument("--model-id", type=str, required=True, help="Model ID for litellm (e.g. openai/Qwen/Qwen2.5-1.5B-Instruct)") parser.add_argument("--max-tokens", type=int, default=128, help="Max tokens for completion") parser.add_argument("--verbose", action="store_true", default=False) args = parser.parse_args() # Create the vLLM model with api_base and kwargs on the Model instance vllm_model = pz.Model(args.model_id, api_base=args.api_base, max_tokens=args.max_tokens) # Load the enron-tiny dataset data_path = os.path.join(os.path.dirname(__file__), "..", "testdata", "enron-tiny") dataset = pz.TextFileDataset(id="test-sentiment", path=data_path) dataset = dataset.sem_map(SentimentResult, desc="Classify the sentiment of the text") # Configure with vLLM model config = pz.QueryProcessorConfig( policy=pz.MaxQuality(), available_models=[vllm_model], execution_strategy="sequential", optimizer_strategy="pareto", verbose=args.verbose, ) output = dataset.run(config) for record in output: print(record) if __name__ == "__main__": main() ================================================ FILE: evals/quest/eval.py ================================================ import argparse import copy import json import os import random import time import palimpzest as pz def prepare_docs_for_query(items: list, gt_docs: list) -> list: items = copy.copy(items) random.shuffle(items) final_items = [doc for doc in items if doc["title"] in gt_docs] while len(final_items) < 1000 and len(items) > 0: item = items.pop(0) if item not in final_items: final_items.append(item) return final_items def palimpzest_run_query(query: dict, documents: list) -> list[str]: gt_docs = query["docs"] items = prepare_docs_for_query(documents, gt_docs) schema = [ {"name": "title", "type": str, "desc": "Document title"}, {"name": "text", "type": str, "desc": "Document content"}, ] dataset = pz.MemoryDataset( id="quest-docs", vals=items, schema=schema, ) query_text = query["query"] plan = dataset.sem_filter( f'This document is relevant to the entity-seeking query: "{query_text}". ' "Return True if the document helps answer the query, False otherwise.", depends_on=["text"], ).project(["title"]) config = pz.QueryProcessorConfig( policy=pz.MaxQuality(), execution_strategy="parallel", progress=True, ) output = plan.run(config) execution_stats = output.execution_stats time_secs = execution_stats.total_execution_time if execution_stats else 0.0 cost = execution_stats.total_execution_cost if execution_stats else 0.0 return [record["title"] for record in output], time_secs, cost def main(): parser = argparse.ArgumentParser(description="Evaluate Palimpzest on QUEST") parser.add_argument( "--domain", type=str, required=True, choices=["films", "books"], help="The domain to evaluate.", ) parser.add_argument( "--queries", type=str, required=True, help="Path to the file containing the queries (e.g. test.jsonl).", ) parser.add_argument( "--documents", type=str, default="data/documents.jsonl", help="Path to documents.jsonl (QUEST format: title, text per line).", ) parser.add_argument( "--limit", type=int, default=None, help="Limit number of queries to evaluate (for debugging).", ) parser.add_argument( "--seed", type=int, default=42, help="Random seed for document shuffling.", ) args = parser.parse_args() random.seed(args.seed) if not os.path.exists(args.documents): raise FileNotFoundError( f"Documents file not found: {args.documents}\n" ) with open(args.documents) as f: documents = [json.loads(line) for line in f] queries = [] with open(args.queries) as f: for line in f: d = json.loads(line) if d["metadata"]["domain"] == args.domain: queries.append(d) if args.limit: queries = queries[: args.limit] results = [] for i, query in enumerate(queries): print(f"[{i + 1}/{len(queries)}] Executing query: {query['query']}") pred_docs, cur_time, cur_cost = palimpzest_run_query(query, documents) gt_docs = query["docs"] preds = set(pred_docs) labels = set(gt_docs) tp = sum(1 for pred in preds if pred in labels) fp = len(preds) - tp fn = sum(1 for label in labels if label not in preds) precision = tp / (tp + fp) if (tp + fp) > 0 else 0 recall = tp / (tp + fn) if (tp + fn) > 0 else 0 f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 result = { "query": query["query"], "predicted_docs": pred_docs, "ground_truth_docs": gt_docs, "precision": precision, "recall": recall, "f1_score": f1, "time": cur_time, "cost": cur_cost } results.append(result) ts = int(time.time()) out_path = f"results_{args.domain}_{ts}.json" with open(out_path, "w") as f: json.dump(results, f, indent=4) print(f"\nResults saved to {out_path}") n = len(results) avg_precision = sum(r["precision"] for r in results) / n avg_recall = sum(r["recall"] for r in results) / n avg_f1 = sum(r["f1_score"] for r in results) / n avg_time = sum(r["time"] for r in results) / n avg_cost = sum(r["cost"] for r in results) / n print(f"Average Precision: {avg_precision:.4f}") print(f"Average Recall: {avg_recall:.4f}") print(f"Average F1 Score: {avg_f1:.4f}") print(f"Average Time: {avg_time:.4f}s") print(f"Average Cost: {avg_cost:.4f}$") if __name__ == "__main__": main() ================================================ FILE: pyproject.toml ================================================ [project] name = "palimpzest" version = "1.5.3" description = "Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language" readme = "README.md" requires-python = ">=3.12" keywords = ["relational", "optimization", "llm", "AI programming", "extraction", "tools", "document", "search", "integration"] authors = [ {name="MIT DSG Semantic Management Lab", email="michjc@csail.mit.edu"}, ] dependencies = [ "anthropic>=0.79.0", "beautifulsoup4>=4.13.4", "chromadb>=1.0.15", "colorama>=0.4.6", "datasets>=4.0.0", "fastapi>=0.115.0", "google-genai>=1.0.0", "gradio>=5.26.0", "litellm>=1.81.11, <1.82.7", "numpy==2.0.2", "openai>=1.0", "pandas>=2.1.1", "pytest>=8.2.2", "pillow>=11.3.0", "prettytable>=3.9.0", "psutil==5.9.5", "PyLD>=2.0.4", "pyarrow>=20.0.0", "pypdf>=5.1.0", "pytest-mock>=3.14.0", "python-dotenv>=1.2.1", "pyyaml>=6.0.1", "requests>=2.25", "ruff>=0.9.0", "sentence-transformers==5.0.0", "setuptools>=70.1.1", "smolagents[toolkit]", "tabulate>=0.9.0", "together>=1.5.5", "tqdm~=4.66.1", "rich[jupyter]>=13.9.2", ] classifiers=[ "Development Status :: 4 - Beta", # Change as appropriate "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", # Change as appropriate "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", # Specify versions you support # Add more classifiers as appropriate ] [project.optional-dependencies] vllm = [ "vllm>=0.10.1.1", ] [tool.setuptools] package-dir = {"" = "src"} include-package-data = true [tool.setuptools.packages.find] where = ["src"] namespaces = false [tool.setuptools.package-data] "*" = ["*.txt", "*.rst", "*.md", "*.json"] [tool.pytest.ini_options] testpaths = ["tests/pytest"] filterwarnings = [ "error", "ignore::DeprecationWarning", "ignore::ResourceWarning", "ignore::UserWarning", ] [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [project.urls] homepage = "https://palimpzest.org" repository = "https://github.com/mitdbg/palimpzest/" documentation = "https://palimpzest.org" # changelog = "https://github.com/me/spam/blob/master/CHANGELOG.md" ================================================ FILE: quickstart.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "dBfyB-7Hytwy" }, "source": [ "![PZ-banner](https://palimpzest-workloads.s3.us-east-1.amazonaws.com/palimpzest-cropped.png)\n", "\n", "# Palimpzest Quickstart\n", "This notebook contains a sample program to guide you through the features of the Palimpzest (PZ) library. PZ provides a high-level, declarative interface for composing and executing pipelines of semantic operators." ] }, { "cell_type": "markdown", "metadata": { "id": "2-TkUeCFx1et" }, "source": [ "## Load Private Key(s)\n", "1. Click on the \"key\" icon on the left-hand-side of the Colab notebook.\n", "2. In the sidebar that opens, click `+ Add new secret`\n", " - **Note:** your secrets are not visible to anyone other than Google and your version of the notebook.\n", "3. Enter one or more of the following keys as secrets:\n", " - `OPENAI_API_KEY`\n", " - `TOGETHER_API_KEY`\n", " - You can create a `together.ai` API key [here](https://api.together.ai/) for this demo (it comes with $1 of free API requests)\n", "4. Make sure you have toggled `Notebook access` ON\n", "5. Execute the cell below to store these keys in notebook environment variables.\n" ] }, { "cell_type": "markdown", "metadata": { "id": "zmmkh1n8efxA" }, "source": [ "#### Note: for the changes to take effect, you may need to restart the session (`Runtime > Restart Session`) if you've already connected the notebook to a runtime" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "-DgUrHNtZu0z" }, "outputs": [], "source": [ "from google.colab import userdata\n", "import os\n", "\n", "# set environment variables\n", "def set_api_key_from_secret(key_name):\n", " try:\n", " os.environ[key_name] = userdata.get(key_name)\n", " except:\n", " pass\n", "\n", "set_api_key_from_secret('OPENAI_API_KEY')\n", "set_api_key_from_secret('TOGETHER_API_KEY')" ] }, { "cell_type": "markdown", "metadata": { "id": "HNFA4gTzxvE2" }, "source": [ "## Install Palimpzest\n", "First, let's install the Palimpzest package. This may take a few minutes. **PIP dependency error messages are expected and can be ignored.**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "id": "4AxQGqXIyXsP" }, "outputs": [], "source": [ "!pip install palimpzest==0.7.6\n", "!pip install --upgrade pyarrow\n", "!pip install chromadb==0.6.3\n", "import palimpzest as pz" ] }, { "cell_type": "markdown", "metadata": { "id": "qSAC96Rb-Ggy" }, "source": [ "## Download Test Files" ] }, { "cell_type": "markdown", "metadata": { "id": "cUwsu8XOzgJd" }, "source": [ "Next, we'll download the dataset we need for this demo:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "id": "IXv-pxMhx0i1" }, "outputs": [], "source": [ "# download tar files with testdata\n", "!wget -nc https://people.csail.mit.edu/gerarvit/PalimpzestData/enron-tiny.tar.gz\n", "!wget -nc wget -nc https://people.csail.mit.edu/gerarvit/PalimpzestData/real-estate-eval-5.tar.gz\n", "!wget -nc https://palimpzest-workloads.s3.us-east-1.amazonaws.com/chroma-biodex.tar.gz\n", "\n", "# open tar files\n", "!tar -xzf enron-tiny.tar.gz\n", "!tar -xzf real-estate-eval-5.tar.gz\n", "!tar -xzf chroma-biodex.tar.gz" ] }, { "cell_type": "markdown", "metadata": { "id": "fw5mmyAY_EaS" }, "source": [ "# First PZ Program: Filtering Enron Emails\n", "For this demo, we will work with a small subset of the Enron Email Dataset to identify emails matching some search criteria.\n", "\n", "We are going to use Palimpzest to perform the following tasks:\n", "1. Load the text files that contain the emails. (Each `.txt` file contains a single email).\n", "2. Compute the sender, subject, and date of each email.\n", "3. Filter the emails for ones that mention a vacation plan and were sent in the month of July.\n", "\n", "We can compose these tasks into a PZ program as follows:\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "id": "lE8xx1s7xoQQ" }, "outputs": [], "source": [ "# define the fields we wish to compute\n", "email_cols = [\n", " {\"name\": \"sender\", \"type\": str, \"desc\": \"The email address of the sender\"},\n", " {\"name\": \"subject\", \"type\": str, \"desc\": \"The subject of the email\"},\n", " {\"name\": \"date\", \"type\": str, \"desc\": \"The date the email was sent\"},\n", "]\n", "\n", "# lazily construct the computation to get emails about holidays sent in July\n", "dataset = pz.Dataset(\"enron-tiny/\")\n", "dataset = dataset.sem_add_columns(email_cols)\n", "dataset = dataset.sem_filter(\"The email was sent in July\")\n", "dataset = dataset.sem_filter(\"The email is about holidays\")" ] }, { "cell_type": "markdown", "metadata": { "id": "ZRYDgD3RsMCo" }, "source": [ "First, we define the set of columns we want to compute in `email_cols`.\n", "\n", "Next, we create a dataset by simply constructing `pz.Dataset()` with to the path to our files.\n", "\n", "We then instruct PZ to compute the email columns with a call to `sem_add_columns()`.\n", "\n", "Finally, we apply our two natural language filters with `sem_filter()`.\n", "\n", "**Note:** due to PZ's lazy execution, the code above will not execute the PZ program. It simply defines the semantic computation graph.\n", "\n", "In the next cell, we execute the PZ program with the goal of optimizing for quality:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cSbS7uC7tUyS" }, "outputs": [], "source": [ "# execute the computation w/the MaxQuality policy\n", "config = pz.QueryProcessorConfig(policy=pz.MaxQuality(), execution_strategy=\"parallel\", progress=True)\n", "output = dataset.run(config)" ] }, { "cell_type": "markdown", "metadata": { "id": "9hAjI6JrtbIU" }, "source": [ "Once our pipeline completes, we can convert the output to a Pandas DataFrame:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cyTrabGGtaZq" }, "outputs": [], "source": [ "# display output (if using Jupyter, otherwise use print(output_df))\n", "output_df = output.to_df(cols=[\"date\", \"sender\", \"subject\"])\n", "display(output_df)" ] }, { "cell_type": "markdown", "metadata": { "id": "55DHU5XNAYBR" }, "source": [ "Furthermore, Palimpzest provides a detailed report of the execution, with statistics about the runtime and cost of each operation, as well as the final plan that PZ executed.\n", "\n", "These statistics are stored in `output.execution_stats`:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ottmnW4OAhXv" }, "outputs": [], "source": [ "print(f\"Optimization Time: {output.execution_stats.optimization_time:.2f}s\")\n", "print(f\"Optimization Cost: ${output.execution_stats.optimization_cost:.3f}\")\n", "print(\"---\")\n", "print(f\"Plan Execution Time: {output.execution_stats.plan_execution_time:.2f}s\")\n", "print(f\"Plan Execution Cost: ${output.execution_stats.plan_execution_cost:.3f}\")\n", "\n", "print(\"Final plan executed:\")\n", "print(\"---\")\n", "final_plan_id = list(output.execution_stats.plan_strs.keys())[-1]\n", "print(output.execution_stats.plan_strs[final_plan_id])" ] }, { "cell_type": "markdown", "metadata": { "id": "ojm-qRxMyO0i" }, "source": [ "# Second PZ Program: Multi-Modal Data Processing\n", "\n", "For our next demo, we will work with a small dataset of five real estate listings to search for properties of interest.\n", "\n", "We are going to use Palimpzest to execute the following pipeline.\n", "1. Load the images and text description for each listing\n", "2. Compute the price and address of each listing from the text description\n", "3. Filter for homes within our price range\n", "4. Filter for homes that look modern and attractive\n", "\n", "Let's take a moment to visualize the homes in our dataset:\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "zUHkZDwC6EdC" }, "outputs": [], "source": [ "from PIL import Image\n", "import numpy as np\n", "import gradio as gr\n", "\n", "# Boilerplate code to build our visualization\n", "fst_imgs, snd_imgs, thrd_imgs, texts = [], [], [], []\n", "for idx in range(1, 6):\n", " listing = f\"listing{idx}\"\n", " with open(os.path.join(\"real-estate-eval-5\", listing, \"listing-text.txt\")) as f:\n", " texts.append(f.read())\n", " for idx, img_name in enumerate([\"img1.png\", \"img2.png\", \"img3.png\"]):\n", " path = os.path.join(\"real-estate-eval-5\", listing, img_name)\n", " img = Image.open(path)\n", " img_arr = np.asarray(img)\n", " if idx == 0:\n", " fst_imgs.append(img_arr)\n", " elif idx == 1:\n", " snd_imgs.append(img_arr)\n", " elif idx == 2:\n", " thrd_imgs.append(img_arr)\n", "\n", "with gr.Blocks() as demo:\n", " fst_img_blocks, snd_img_blocks, thrd_img_blocks, text_blocks = [], [], [], []\n", " for fst_img, snd_img, thrd_img, text in zip(fst_imgs, snd_imgs, thrd_imgs, texts):\n", " with gr.Row(equal_height=True):\n", " with gr.Column():\n", " fst_img_blocks.append(gr.Image(value=fst_img))\n", " with gr.Column():\n", " snd_img_blocks.append(gr.Image(value=snd_img))\n", " with gr.Column():\n", " thrd_img_blocks.append(gr.Image(value=thrd_img))\n", " with gr.Row():\n", " with gr.Column():\n", " text_blocks.append(gr.Textbox(value=text, info=\"Text Description\"))\n", "\n", "demo.launch()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Yg4yRYx26ecr" }, "outputs": [], "source": [ "demo.close()" ] }, { "cell_type": "markdown", "metadata": { "id": "GpNe3bFk6FMD" }, "source": [ "As a first step, we need to write a custom `pz.DataReader` to enable PZ to load our data properly:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "2FcqpZySyWbl" }, "outputs": [], "source": [ "from palimpzest.core.lib.fields import ImageFilepathField, ListField\n", "\n", "# we first define the schema for each record output by the DataReader\n", "real_estate_listing_cols = [\n", " {\"name\": \"listing\", \"type\": str, \"desc\": \"The name of the listing\"},\n", " {\"name\": \"text_content\", \"type\": str, \"desc\": \"The content of the listing's text description\"},\n", " {\"name\": \"image_filepaths\", \"type\": ListField(ImageFilepathField), \"desc\": \"A list of the filepaths for each image of the listing\"},\n", "]\n", "\n", "# we then implement the DataReader\n", "class RealEstateListingReader(pz.DataReader):\n", " def __init__(self, listings_dir):\n", " super().__init__(schema=real_estate_listing_cols)\n", " self.listings_dir = listings_dir\n", " self.listings = sorted(os.listdir(self.listings_dir))\n", "\n", " def __len__(self):\n", " return len(self.listings)\n", "\n", " def __getitem__(self, idx: int):\n", " # get listing\n", " listing = self.listings[idx]\n", "\n", " # get fields\n", " image_filepaths, text_content = [], None\n", " listing_dir = os.path.join(self.listings_dir, listing)\n", " for file in os.listdir(listing_dir):\n", " if file.endswith(\".txt\"):\n", " with open(os.path.join(listing_dir, file), \"rb\") as f:\n", " text_content = f.read().decode(\"utf-8\")\n", " elif file.endswith(\".png\"):\n", " image_filepaths.append(os.path.join(listing_dir, file))\n", "\n", " # construct and return dictionary with fields\n", " return {\"listing\": listing, \"text_content\": text_content, \"image_filepaths\": image_filepaths}" ] }, { "cell_type": "markdown", "metadata": { "id": "Q45qFYcM0N9w" }, "source": [ "Every `pz.DataReader` must have the following:\n", "1. A `schema` defining the fields present in each output record\n", "2. A `__len__()` function which returns the number of items in the dataset\n", "3. A `__getitem__(idx)` function which returns the `idx`th item in the dataset\n", "\n", "Once we've implemented the `pz.DataReader`, we can compose our PZ program as follows:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1qOI9WOY0X4_" }, "outputs": [], "source": [ "# schema for computing the address and price of each home\n", "real_estate_text_cols = [\n", " {\"name\": \"address\", \"type\": str, \"desc\": \"The address of the property\"},\n", " {\"name\": \"price\", \"type\": int | float, \"desc\": \"The listed price of the property\"},\n", "]\n", "\n", "# define a UDF for filtering based on a price range\n", "def in_price_range(record: dict):\n", " try:\n", " price = record[\"price\"]\n", " if isinstance(price, str):\n", " price = price.strip()\n", " price = int(price.replace(\"$\", \"\").replace(\",\", \"\"))\n", " return 6e5 < price <= 2e6\n", " except Exception:\n", " return False\n", "\n", "# construct our PZ program to filter for listings matching our search criteria\n", "ds = pz.Dataset(RealEstateListingReader(\"real-estate-eval-5\"))\n", "ds = ds.sem_add_columns(real_estate_text_cols, depends_on=\"text_content\")\n", "ds = ds.sem_filter(\n", " \"The interior is modern and attractive, and has lots of natural sunlight\",\n", " depends_on=\"image_filepaths\",\n", ")\n", "ds = ds.filter(in_price_range, depends_on=\"price\")" ] }, { "cell_type": "markdown", "metadata": { "id": "eitGXQCS0YRY" }, "source": [ "First, we write a schema for the `address` and `price` fields we wish to compute.\n", "\n", "Next, we write a UDF to filter for homes based on our price range.\n", "\n", "Then we compose our program by:\n", "1. Constructing our `pz.DataReader` with the real estate data\n", "2. Using `sem_add_columns()` to compute the `address` and `price`\n", "3. Using a `sem_filter()` to filter for modern homes with lots of sunlight\n", "4. Using our UDF to filter for homes based on our price range\n", "\n", "We now execute the program:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "uFmakjcQ4W5o" }, "outputs": [], "source": [ "# execute the computation w/the MaxQuality policy\n", "config = pz.QueryProcessorConfig(policy=pz.MaxQuality(), execution_strategy=\"parallel\", progress=True)\n", "output = ds.run(config)" ] }, { "cell_type": "markdown", "metadata": { "id": "YHesFcjc4snL" }, "source": [ "Now let's take a look at our output:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "JAxeR_R54vuY" }, "outputs": [], "source": [ "from PIL import Image\n", "import numpy as np\n", "import gradio as gr\n", "\n", "demo.close()\n", "\n", "# Boilerplate code to build our visualization\n", "fst_imgs, snd_imgs, thrd_imgs, addrs, prices = [], [], [], [], []\n", "for record in output:\n", " addrs.append(record.address)\n", " prices.append(record.price)\n", " for idx, img_name in enumerate([\"img1.png\", \"img2.png\", \"img3.png\"]):\n", " path = os.path.join(\"real-estate-eval-5\", record.listing, img_name)\n", " img = Image.open(path)\n", " img_arr = np.asarray(img)\n", " if idx == 0:\n", " fst_imgs.append(img_arr)\n", " elif idx == 1:\n", " snd_imgs.append(img_arr)\n", " elif idx == 2:\n", " thrd_imgs.append(img_arr)\n", "\n", "with gr.Blocks() as demo:\n", " fst_img_blocks, snd_img_blocks, thrd_img_blocks, addr_blocks, price_blocks = [], [], [], [], []\n", " for fst_img, snd_img, thrd_img, addr, price in zip(fst_imgs, snd_imgs, thrd_imgs, addrs, prices):\n", " with gr.Row(equal_height=True):\n", " with gr.Column():\n", " fst_img_blocks.append(gr.Image(value=fst_img))\n", " with gr.Column():\n", " snd_img_blocks.append(gr.Image(value=snd_img))\n", " with gr.Column():\n", " thrd_img_blocks.append(gr.Image(value=thrd_img))\n", " with gr.Row():\n", " with gr.Column():\n", " addr_blocks.append(gr.Textbox(value=addr, info=\"Address\"))\n", " with gr.Column():\n", " price_blocks.append(gr.Textbox(value=price, info=\"Price\"))\n", "\n", " plan_str = list(output.execution_stats.plan_strs.values())[0]\n", " gr.Textbox(value=plan_str, info=\"Query Plan\")\n", "\n", "demo.launch()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XpCo8uF54sPi" }, "outputs": [], "source": [ "demo.close()" ] }, { "cell_type": "markdown", "metadata": { "id": "VAgG4wBKyZM0" }, "source": [ "# Third PZ Program: Optimizing a Biomedical Classification Pipeline\n", "\n", "For our final demo, we will work with a subset of the BioDEX dataset.\n", "\n", "Each input in the dataset is a medical report describing an adverse reaction a patient had in response to taking one or more drugs.\n", "\n", "The goal is to correctly predict the reactions experienced by the patient by matching them to a database of ~24,300 official medical reaction terms.\n", "\n", "We are going to use Palimpzest to implement the following pipeline:\n", "1. Load a medical report\n", "2. Compute a list of reactions mentioned in the report\n", "3. Retrieve the most similar reaction terms from a vector database with embeddings for each of the ~24,300 official terms\n", "4. Re-rank the list of official terms based on their relevance\n", "\n", "First, we will once again create a `pz.DataReader` to load the medical reports:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "_CeZ9Ib1yY1n" }, "outputs": [], "source": [ "import datasets\n", "from functools import partial\n", "\n", "# define the schema for records returned by the DataReader\n", "biodex_entry_cols = [\n", " {\"name\": \"pmid\", \"type\": str, \"desc\": \"The PubMed ID of the medical paper\"},\n", " {\"name\": \"title\", \"type\": str, \"desc\": \"The title of the medical paper\"},\n", " {\"name\": \"abstract\", \"type\": str, \"desc\": \"The abstract of the medical paper\"},\n", " {\"name\": \"fulltext\", \"type\": str, \"desc\": \"The full text of the medical paper, which contains information relevant for creating a drug safety report.\"},\n", "]\n", "\n", "# implement the DataReader\n", "class BiodexReader(pz.DataReader):\n", " def __init__(\n", " self,\n", " rp_at_k: int = 5,\n", " num_samples: int = 10,\n", " split: str = \"test\",\n", " shuffle: bool = True,\n", " seed: int = 42,\n", " ):\n", " super().__init__(biodex_entry_cols)\n", "\n", " self.dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=split).to_pandas()\n", " if shuffle:\n", " self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient=\"records\")\n", " else:\n", " self.dataset = self.dataset.to_dict(orient=\"records\")[:num_samples]\n", "\n", " self.rp_at_k = rp_at_k\n", " self.num_samples = num_samples\n", " self.shuffle = shuffle\n", " self.seed = seed\n", " self.split = split\n", "\n", " def compute_label(self, entry: dict) -> dict:\n", " \"\"\"Compute the label for a BioDEX report given its entry in the dataset.\"\"\"\n", " reactions_lst = [\n", " reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n", " for reaction in entry[\"reactions\"].split(\",\")\n", " ]\n", " label_dict = {\n", " \"reactions\": reactions_lst,\n", " \"reaction_labels\": reactions_lst,\n", " \"ranked_reaction_labels\": reactions_lst,\n", " }\n", " return label_dict\n", "\n", " @staticmethod\n", " def rank_precision_at_k(preds, targets, k: int):\n", " if preds is None:\n", " return 0.0\n", "\n", " try:\n", " # lower-case each list\n", " preds = [pred.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for pred in preds]\n", " targets = set([target.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for target in targets])\n", "\n", " # compute rank-precision at k\n", " rn = len(targets)\n", " denom = min(k, rn)\n", " total = 0.0\n", " for i in range(k):\n", " total += preds[i] in targets if i < len(preds) else 0.0\n", "\n", " return total / denom\n", "\n", " except Exception:\n", " return 0.0\n", "\n", " @staticmethod\n", " def term_recall(preds, targets):\n", " if preds is None:\n", " return 0.0\n", "\n", " try:\n", " # normalize terms in each list\n", " pred_terms = set([\n", " term.strip()\n", " for pred in preds\n", " for term in pred.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n", " ])\n", " target_terms = ([\n", " term.strip()\n", " for target in targets\n", " for term in target.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n", " ])\n", "\n", " # compute term recall and return\n", " intersect = pred_terms.intersection(target_terms)\n", " term_recall = len(intersect) / len(target_terms)\n", "\n", " return term_recall\n", "\n", " except Exception:\n", " return 0.0\n", "\n", " def __len__(self):\n", " return len(self.dataset)\n", "\n", " def __getitem__(self, idx: int):\n", " # get entry\n", " entry = self.dataset[idx]\n", "\n", " # get input fields\n", " pmid = entry[\"pmid\"]\n", " title = entry[\"title\"]\n", " abstract = entry[\"abstract\"]\n", " fulltext = entry[\"fulltext\"]\n", "\n", " # create item with fields\n", " item = {\"fields\": {}, \"labels\": {}, \"score_fn\": {}}\n", " item[\"fields\"][\"pmid\"] = pmid\n", " item[\"fields\"][\"title\"] = title\n", " item[\"fields\"][\"abstract\"] = abstract\n", " item[\"fields\"][\"fulltext\"] = fulltext\n", "\n", " if self.split == \"train\":\n", " # add label info\n", " item[\"labels\"] = self.compute_label(entry)\n", "\n", " # add scoring functions for list fields\n", " rank_precision_at_k = partial(BiodexReader.rank_precision_at_k, k=self.rp_at_k)\n", " item[\"score_fn\"][\"reactions\"] = BiodexReader.term_recall\n", " item[\"score_fn\"][\"reaction_labels\"] = BiodexReader.term_recall\n", " item[\"score_fn\"][\"ranked_reaction_labels\"] = rank_precision_at_k\n", "\n", " return item\n" ] }, { "cell_type": "markdown", "metadata": { "id": "fyr71cs3EQpl" }, "source": [ "There are a few new features of this `pz.DataReader` which are needed for the optimization process:\n", "1. `__getitem__()` returns a dictionary with top-level keys `{\"fields\", \"labels\", \"score_fn\"}`\n", "2. `fields` contains the data emitted by the `pz.DataReader`\n", "3. (for `train` data only): `labels` contains the expected results for each output field\n", "4. (for `train` data only): `score_fn` contains scoring functions for each output field\n", "\n", "Once we've defined our `pz.DataReader`, we can create our training and test datasets:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "aHYqdDrlG8zP" }, "outputs": [], "source": [ "SEED = 123\n", "\n", "# create train and test datasets; and validator\n", "train_datareader = BiodexReader(split=\"train\", seed=SEED)\n", "test_datareader = BiodexReader(split=\"test\", num_samples=20, seed=SEED)\n", "validator = pz.Validator(train_datareader, None)" ] }, { "cell_type": "markdown", "metadata": { "id": "lHaVbQHiG-Rf" }, "source": [ "We now implement the logic for the `sem_topk` operator for you. It fetches the five most similar medical terms for each reaction computed by PZ, sorts them based on similarity, and then returns the final top-k most similar terms." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "JE3scuaXI6HB" }, "outputs": [], "source": [ "import chromadb\n", "from chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction\n", "\n", "# load index [text-embedding-3-small]\n", "chroma_client = chromadb.PersistentClient(\".chroma-biodex\")\n", "openai_ef = OpenAIEmbeddingFunction(\n", " api_key=os.environ[\"OPENAI_API_KEY\"],\n", " model_name=\"text-embedding-3-small\",\n", ")\n", "index = chroma_client.get_collection(\"biodex-reaction-terms\", embedding_function=openai_ef)\n", "\n", "def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]:\n", " # execute query with embeddings\n", " results = index.query(query, n_results=5)\n", "\n", " # get list of result terms with their cosine similarity scores\n", " final_results = []\n", " for query_docs, query_distances in zip(results[\"documents\"], results[\"distances\"]):\n", " for doc, dist in zip(query_docs, query_distances):\n", " cosine_similarity = 1 - dist\n", " final_results.append({\"content\": doc, \"similarity\": cosine_similarity})\n", "\n", " # sort the results by similarity score\n", " sorted_results = sorted(final_results, key=lambda result: result[\"similarity\"], reverse=True)\n", "\n", " # remove duplicates\n", " sorted_results_set = set()\n", " final_sorted_results = []\n", " for result in sorted_results:\n", " if result[\"content\"] not in sorted_results_set:\n", " sorted_results_set.add(result[\"content\"])\n", " final_sorted_results.append(result[\"content\"])\n", "\n", " # return the top-k similar results and generation stats\n", " return {\"reaction_labels\": final_sorted_results[:k]}" ] }, { "cell_type": "markdown", "metadata": { "id": "dCYi92YCJSuq" }, "source": [ "Finally, we can construct our PZ program:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Z3UkmjQFBSjc" }, "outputs": [], "source": [ "# define the schema for each computation in our program\n", "biodex_reactions_cols = [\n", " {\"name\": \"reactions\", \"type\": list[str], \"desc\": \"The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible.\"},\n", "]\n", "biodex_reaction_labels_cols = [\n", " {\"name\": \"reaction_labels\", \"type\": list[str], \"desc\": \"Official terms for medical conditions listed in `reactions`\"},\n", "]\n", "biodex_ranked_reactions_labels_cols = [\n", " {\"name\": \"ranked_reaction_labels\", \"type\": list[str], \"desc\": \"The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs.\"},\n", "]\n", "\n", "\n", "# construct pz plan\n", "plan = pz.Dataset(test_datareader)\n", "plan = plan.sem_add_columns(biodex_reactions_cols)\n", "plan = plan.sem_topk(\n", " index=index,\n", " search_func=search_func,\n", " search_attr=\"reactions\",\n", " output_attrs=biodex_reaction_labels_cols,\n", ")\n", "plan = plan.sem_add_columns(biodex_ranked_reactions_labels_cols, depends_on=[\"title\", \"abstract\", \"fulltext\", \"reaction_labels\"])\n" ] }, { "cell_type": "markdown", "metadata": { "id": "yvFZO-LKKq3F" }, "source": [ "First, let's execute our plan without training data and score our performance:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "khhJJUXTLNDz" }, "outputs": [], "source": [ "def score_output(output, seed):\n", " # score output\n", " test_dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=\"test\").to_pandas()\n", " test_dataset = test_dataset.sample(n=20, random_state=seed).to_dict(orient=\"records\")\n", "\n", " # construct mapping from pmid --> label (field, value) pairs\n", " def compute_target_record(entry):\n", " reactions_lst = [\n", " reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n", " for reaction in entry[\"reactions\"].split(\",\")\n", " ]\n", " label_dict = {\"ranked_reaction_labels\": reactions_lst}\n", " return label_dict\n", "\n", " label_fields_to_values = {\n", " entry[\"pmid\"]: compute_target_record(entry) for entry in test_dataset\n", " }\n", "\n", " def rank_precision_at_k(preds: list, targets: list, k: int):\n", " if preds is None:\n", " return 0.0\n", "\n", " # lower-case each list\n", " preds = [pred.lower().replace(\"'\", \"\").replace(\"^\", \"\") for pred in preds]\n", " targets = set([target.lower().replace(\"'\", \"\").replace(\"^\", \"\") for target in targets])\n", "\n", " # compute rank-precision at k\n", " rn = len(targets)\n", " denom = min(k, rn)\n", " total = 0.0\n", " for i in range(k):\n", " total += preds[i] in targets if i < len(preds) else 0.0\n", "\n", " return total / denom\n", "\n", " def compute_avg_rp_at_k(records, k=5):\n", " total_rp_at_k = 0\n", " bad = 0\n", " for record in records:\n", " pmid = record['pmid']\n", " preds = record['ranked_reaction_labels']\n", " targets = label_fields_to_values[pmid]['ranked_reaction_labels']\n", " try:\n", " total_rp_at_k += rank_precision_at_k(preds, targets, k)\n", " except Exception:\n", " bad += 1\n", "\n", " return total_rp_at_k / len(records), bad\n", "\n", " rp_at_k, bad = compute_avg_rp_at_k([record.to_dict() for record in output], k=5)\n", " final_plan_id = list(output.execution_stats.plan_stats.keys())[0]\n", " final_plan_str = output.execution_stats.plan_strs[final_plan_id]\n", " print(\"---\")\n", " print(\"#########################\")\n", " print(f\"##### RP@5: {rp_at_k:.5f} #####\")\n", " print(\"#########################\")\n", " print(\"---\")\n", " print(f\"Optimization time: {output.execution_stats.optimization_time:.2f}s\")\n", " print(f\"Optimization cost: ${output.execution_stats.optimization_cost:.3f}\")\n", " print(\"---\")\n", " print(f\"Plan exec. time: {output.execution_stats.plan_execution_time:.2f}s\")\n", " print(f\"Plan exec. cost: ${output.execution_stats.plan_execution_cost:.3f}\")\n", " print(\"---\")\n", " print(f\"Total time: {output.execution_stats.total_execution_time:.2f}s\")\n", " print(f\"Total Cost: ${output.execution_stats.total_execution_cost:.3f}\")\n", " print(\"---\")\n", " print(\"Final Plan:\")\n", " print(final_plan_str)\n", "\n", "import logging\n", "logger = logging.getLogger()\n", "logger.disabled = True\n", "\n", "# execute pz plan\n", "config = pz.QueryProcessorConfig(\n", " policy=pz.MaxQuality(),\n", " execution_strategy=\"parallel\",\n", " max_workers=64,\n", " progress=True,\n", ")\n", "\n", "output = plan.run(config=config, seed=SEED)\n", "score_output(output, seed=SEED)" ] }, { "cell_type": "markdown", "metadata": { "id": "ORUDS35-K1Nf" }, "source": [ "Now, let's run the program again while using our `train_datareader` as a validation dataset:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "a4htdaTDKpFc" }, "outputs": [], "source": [ "import logging\n", "logger = logging.getLogger()\n", "logger.disabled = True\n", "\n", "# execute pz plan\n", "config = pz.QueryProcessorConfig(\n", " policy=pz.MaxQuality(),\n", " validator=validator,\n", " optimizer_strategy=\"pareto\",\n", " sentinel_execution_strategy=\"mab\",\n", " execution_strategy=\"parallel\",\n", " use_final_op_quality=True,\n", " max_workers=64,\n", " progress=True,\n", ")\n", "\n", "output = plan.run(config=config, k=6, j=4, sample_budget=72, seed=SEED)\n", "score_output(output, seed=SEED)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "IT3iZr6-Kpab" }, "outputs": [], "source": [] } ], "metadata": { "colab": { "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: ruff.toml ================================================ # Config https://docs.astral.sh/ruff/configuration/ line-length = 120 indent-width = 4 exclude = ["*.ipynb"] # Assume Python 3.8 target-version = "py38" [lint] ignore = ["E501"] fixable = ["ALL"] unfixable = [] select = ["E", "F", "UP", "B", "SIM", "I", "N"] ================================================ FILE: scripts/capture_litellm_stats.py ================================================ #!/usr/bin/env python3 """ Script to invoke LLM providers through LiteLLM and capture token/cost statistics. This script: 1. Loads messages from JSON files generated by generate_test_messages.py 2. Sends requests through LiteLLM (the same path palimpzest uses) 3. Saves all usage metadata and response stats returned by LiteLLM 4. Waits 10 seconds 5. Sends the request again and saves the second set of stats This allows us to compare LiteLLM's reported statistics with: - Direct provider API calls (from capture_provider_stats.py) - Palimpzest's generator stats tracking Supported providers: - Anthropic: claude-sonnet-4-5-20250929 (text, image, text+image) - Google/Gemini: gemini-2.5-flash (all seven modality combinations) - OpenAI: gpt-4o-2024-08-06 (text, image, text+image) - OpenAI: gpt-4o-audio-preview (text+audio, audio) - Azure: gpt-4o via Azure OpenAI (text, image, text+image) Output files are saved to: scripts/litellm_stats/ """ import argparse import json import os import sys import time import uuid from typing import Any import litellm from litellm.integrations.custom_logger import CustomLogger # Add project root to path sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) import contextlib from palimpzest.constants import Model # ============================================================================= # RAW RESPONSE CAPTURE CALLBACK # ============================================================================= class RawProviderStatsCapture(CustomLogger): """ Custom LiteLLM callback to capture raw provider usage stats before normalization. LiteLLM normalizes all responses to OpenAI format, which loses provider-specific details like Gemini's per-modality token breakdowns. This callback captures the original provider response data. """ def __init__(self): self.last_raw_response = None self.last_raw_usage = None self.last_provider = None def log_success_event(self, kwargs, response_obj, start_time, end_time): """Called after a successful LLM API call.""" try: # Store the provider info self.last_provider = kwargs.get("custom_llm_provider") or kwargs.get("model", "").split("/")[0] # Try to get the original response from hidden params if hasattr(response_obj, "_hidden_params") and response_obj._hidden_params: hidden = response_obj._hidden_params self.last_raw_response = hidden.get("original_response") # For some providers, the raw response might be in different locations if self.last_raw_response is None: self.last_raw_response = hidden.get("raw_response") # Try to extract raw usage from the response object itself # Some providers have additional attributes that aren't in model_dump() if hasattr(response_obj, "_response_ms"): if self.last_raw_response is None: self.last_raw_response = {} self.last_raw_response["_response_ms"] = response_obj._response_ms # For Vertex AI / Gemini, check for provider-specific usage fields if hasattr(response_obj, "vertex_ai_usage_metadata"): self.last_raw_usage = response_obj.vertex_ai_usage_metadata elif hasattr(response_obj, "_vertex_ai_response"): self.last_raw_response = response_obj._vertex_ai_response except Exception as e: # Don't let callback errors break the main flow print(f" [Callback] Error capturing raw response: {e}") def log_failure_event(self, kwargs, response_obj, start_time, end_time): """Called after a failed LLM API call.""" self.last_raw_response = None self.last_raw_usage = None self.last_provider = None def reset(self): """Reset captured data for next request.""" self.last_raw_response = None self.last_raw_usage = None self.last_provider = None def get_captured_data(self) -> dict[str, Any]: """Return captured raw data and reset for next request.""" data = { "raw_provider_response": self.last_raw_response, "raw_provider_usage": self.last_raw_usage, "detected_provider": self.last_provider, } self.reset() return data # Global callback instance raw_stats_capture = RawProviderStatsCapture() # Register the callback with LiteLLM litellm.callbacks = [raw_stats_capture] # Enable return of response headers (helps with some providers) litellm.return_response_headers = True # ============================================================================= # PROVIDER CONFIGURATIONS # ============================================================================= # Maps provider name to Model enum and supported modalities # The Model enum is used for: # 1. Getting the LiteLLM model name via model.value # 2. Initializing PromptManager which needs a Model enum PROVIDER_MODALITY_SUPPORT = { "anthropic": { "model": Model.CLAUDE_4_5_SONNET, "supported_modalities": ["text-only", "image-only", "text-image"], # Note: Anthropic does not support audio }, "openai": { "model": Model.GPT_4o, "supported_modalities": ["text-only", "image-only", "text-image"], }, "openai-audio": { "model": Model.GPT_4o_AUDIO_PREVIEW, "supported_modalities": ["audio-only", "text-audio"], }, "gemini": { "model": Model.GOOGLE_GEMINI_2_5_FLASH, "supported_modalities": [ "text-only", "image-only", "audio-only", "text-image", "text-audio", "image-audio", "text-image-audio", ], }, "vertex_ai": { "model": Model.GEMINI_2_5_FLASH, "supported_modalities": [ "text-only", "image-only", "audio-only", "text-image", "text-audio", "image-audio", "text-image-audio", ], }, "azure": { "model": Model.AZURE_GPT_4o, "supported_modalities": ["text-only", "image-only", "text-image"], }, } def load_messages(modality: str, provider: str, messages_dir: str) -> list[dict]: """Load messages from JSON file for a given modality/provider combination.""" filepath = os.path.join(messages_dir, f"{modality}_{provider}.json") with open(filepath) as f: return json.load(f) def transform_messages_for_litellm(messages: list[dict]) -> list[dict]: """ Transform palimpzest message format to LiteLLM-compatible format. LiteLLM expects: - Messages with role and content - Content can be string or list of content blocks - No 'type' field at the message level (that's palimpzest-specific) This function consolidates multiple user messages with different types into single messages with combined content. """ litellm_messages = [] for msg in messages: role = msg.get("role") msg_type = msg.get("type") content = msg.get("content") if role == "system": # System messages pass through as-is # Content may be string or list of content blocks (for caching) litellm_messages.append({"role": "system", "content": content}) elif role == "user": # User messages need consolidation if msg_type == "text": # Text content - string or list of content blocks if litellm_messages and litellm_messages[-1]["role"] == "user": # Merge with existing user message existing = litellm_messages[-1]["content"] if isinstance(existing, str): if isinstance(content, str): litellm_messages[-1]["content"] = [ {"type": "text", "text": existing}, {"type": "text", "text": content}, ] else: litellm_messages[-1]["content"] = [ {"type": "text", "text": existing} ] + content else: if isinstance(content, str): existing.append({"type": "text", "text": content}) else: existing.extend(content) else: litellm_messages.append({"role": "user", "content": content}) elif msg_type == "image": # Image content - list of image_url blocks if litellm_messages and litellm_messages[-1]["role"] == "user": existing = litellm_messages[-1]["content"] if isinstance(existing, str): litellm_messages[-1]["content"] = [ {"type": "text", "text": existing} ] + content else: existing.extend(content) else: litellm_messages.append({"role": "user", "content": content}) elif msg_type == "input_audio": # Audio content - list of input_audio blocks if litellm_messages and litellm_messages[-1]["role"] == "user": existing = litellm_messages[-1]["content"] if isinstance(existing, str): litellm_messages[-1]["content"] = [ {"type": "text", "text": existing} ] + content else: existing.extend(content) else: litellm_messages.append({"role": "user", "content": content}) elif role == "assistant": litellm_messages.append({"role": "assistant", "content": content}) return litellm_messages def call_litellm_api( messages: list[dict], model: Model, provider: str, cache_key: str | None = None, ) -> dict[str, Any]: """ Call LiteLLM completion API and return all usage statistics. This function captures both: - Option A: Raw provider usage via callback (if available) - Option C: Normalized LiteLLM usage (fallback) Args: messages: List of message dicts (palimpzest format) model: Model enum (used for model name and provider detection) provider: Provider name for logging cache_key: Optional prompt_cache_key for OpenAI sticky routing to same cache shard Returns dict with: - usage: Normalized usage dict from LiteLLM response (Option C fallback) - usage_raw: Raw provider usage if captured via callback (Option A) - response_content: First 200 chars of response - model: Model used - raw_response: Full response object serialized """ # Reset the callback to capture fresh data for this request raw_stats_capture.reset() # Transform messages to LiteLLM format litellm_messages = transform_messages_for_litellm(messages) # Get the LiteLLM model name from the Model enum model_name = model.value # Set up completion kwargs completion_kwargs = { "temperature": 0.0, } # Add modalities for audio models if "audio" in model_name.lower(): completion_kwargs["modalities"] = ["text"] # Apply provider-specific caching configuration # Messages from generator_messages already have cache_control markers for Anthropic if (model.is_provider_openai() or model.is_provider_azure()) and cache_key: # OpenAI: Use prompt_cache_key for sticky routing to the same cache shard # https://platform.openai.com/docs/guides/prompt-caching completion_kwargs["extra_body"] = {"prompt_cache_key": cache_key} # Make the LiteLLM call response = litellm.completion( model=model_name, messages=litellm_messages, **completion_kwargs, ) # ========================================================================== # Option C (Fallback): Extract normalized usage stats from LiteLLM response # ========================================================================== usage_normalized = {} if response.usage: usage_normalized = response.usage.model_dump() # ========================================================================== # Option A: Get raw provider data captured by callback # ========================================================================== callback_data = raw_stats_capture.get_captured_data() usage_raw = callback_data.get("raw_provider_usage") # Also try to extract raw usage from _hidden_params hidden_params = {} try: if hasattr(response, "_hidden_params") and response._hidden_params: hidden_params = dict(response._hidden_params) # Some providers store original response here if "original_response" in hidden_params: original = hidden_params["original_response"] if isinstance(original, dict) and "usage_metadata" in original: usage_raw = original["usage_metadata"] elif hasattr(original, "usage_metadata"): with contextlib.suppress(Exception): usage_raw = original.usage_metadata.model_dump() if hasattr(original.usage_metadata, "model_dump") else dict(original.usage_metadata) except Exception: pass # Get response text safely try: response_text = ( response.choices[0].message.content[:200] if response.choices and response.choices[0].message.content else None ) except Exception: response_text = None # Serialize the full response for debugging try: raw_response = response.model_dump() except Exception: raw_response = str(response) return { "provider": provider, "model": model_name, "usage": usage_normalized, # Option C: Normalized LiteLLM format "usage_raw": usage_raw, # Option A: Raw provider format (if captured) "response_content": response_text, "raw_response": raw_response, "hidden_params": hidden_params, "callback_data": callback_data, } def capture_stats_for_provider( provider: str, modality: str, messages: list[dict], model: Model, ) -> dict[str, Any]: """ Capture stats for a provider by making two requests with a delay. Args: provider: Provider name (for logging and file naming) modality: Modality name messages: List of message dicts model: Model enum Returns dict with: - first_request: stats from first request - second_request: stats from second request (should show cache hits) """ # Generate a unique cache key for OpenAI (ensures both requests hit the same cache shard) # Reference: capture_provider_stats.py and PromptManager.__init__ openai_cache_key = f"pz-test-{uuid.uuid4().hex[:12]}" if provider in ("openai", "openai-audio", "azure") else None print(" First request...") first_stats = call_litellm_api(messages, model, provider, cache_key=openai_cache_key) print(f" Usage: {first_stats['usage']}") print(" Waiting 20 seconds for cache to be available...") time.sleep(20) print(" Second request (should show cache hits)...") second_stats = call_litellm_api(messages, model, provider, cache_key=openai_cache_key) print(f" Usage: {second_stats['usage']}") return { "provider": provider, "model": model.value, "modality": modality, "first_request": first_stats, "second_request": second_stats, } def save_stats(stats: dict[str, Any], output_dir: str, provider: str, modality: str) -> str: """Save stats to a JSON file.""" os.makedirs(output_dir, exist_ok=True) output_path = os.path.join(output_dir, f"{provider}_{modality}.json") with open(output_path, "w") as f: json.dump(stats, f, indent=2, default=str) return output_path def main(): """Capture LiteLLM stats for supported provider/modality combinations.""" parser = argparse.ArgumentParser( description="Capture token/cost statistics from LLM providers via LiteLLM.", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=f""" Available providers: {', '.join(PROVIDER_MODALITY_SUPPORT.keys())} Available modalities: text-only, image-only, audio-only, text-image, text-audio, image-audio, text-image-audio Examples: python capture_litellm_stats.py # Run all providers/modalities python capture_litellm_stats.py -p openai # Run all modalities for OpenAI python capture_litellm_stats.py -p openai -m text-only # Run only text-only for OpenAI python capture_litellm_stats.py -p anthropic -m text-image # Run only text-image for Anthropic """ ) parser.add_argument( "-p", "--provider", nargs="+", choices=list(PROVIDER_MODALITY_SUPPORT.keys()), help="Provider(s) to run. If not specified, runs all providers.", ) parser.add_argument( "-m", "--modality", nargs="+", choices=["text-only", "image-only", "audio-only", "text-image", "text-audio", "image-audio", "text-image-audio"], help="Modality(ies) to run. If not specified, runs all supported modalities for each provider.", ) args = parser.parse_args() messages_dir = os.path.join( os.path.dirname(__file__), "..", "tests", "pytest", "data", "generator_messages", ) messages_dir = os.path.abspath(messages_dir) output_dir = os.path.join( os.path.dirname(__file__), "litellm_stats", ) output_dir = os.path.abspath(output_dir) print(f"Loading messages from: {messages_dir}") print(f"Saving stats to: {output_dir}\n") # Ensure output directory exists os.makedirs(output_dir, exist_ok=True) # Determine which providers to run providers_to_run = args.provider if args.provider else list(PROVIDER_MODALITY_SUPPORT.keys()) print(f"Providers to run: {providers_to_run}\n") for provider in providers_to_run: config = PROVIDER_MODALITY_SUPPORT[provider] model = config["model"] supported_modalities = config["supported_modalities"] # Filter modalities if specified if args.modality: modalities_to_run = [m for m in args.modality if m in supported_modalities] if not modalities_to_run: print(f"\nProvider: {provider} - SKIPPED (none of {args.modality} supported)") continue else: modalities_to_run = supported_modalities print(f"\nProvider: {provider} (model: {model.value})") print(f" Modalities to run: {modalities_to_run}") for modality in modalities_to_run: print(f"\n Processing modality: {modality}") try: messages = load_messages(modality, provider, messages_dir) print(f" Loaded {len(messages)} messages from {modality}_{provider}.json") stats = capture_stats_for_provider(provider, modality, messages, model) output_path = save_stats(stats, output_dir, provider, modality) print(f" Saved to: {output_path}") except FileNotFoundError as e: print(f" SKIPPED: Message file not found - {e}") except Exception as e: print(f" ERROR: {e}") import traceback traceback.print_exc() print("\nDone!") if __name__ == "__main__": main() ================================================ FILE: scripts/capture_provider_stats.py ================================================ #!/usr/bin/env python3 """ Script to directly invoke LLM providers and capture token/cost statistics. This script: 1. Loads messages from JSON files generated by generate_test_messages.py 2. Sends requests directly to each provider's API (not through litellm) 3. Saves the token/cost related stats returned by the provider 4. Waits 10 seconds 5. Sends the request again and saves the second set of stats This allows us to establish baseline expectations for what the providers return, which can then be used to validate the palimpzest generator's stats tracking. Supported providers: - Anthropic: claude-sonnet-4-5-20250929 (text, image, text+image) - Google/Vertex AI: gemini-2.5-flash (all seven modality combinations) - OpenAI: gpt-4o-2024-08-06 (text, image, text+image) - OpenAI: gpt-4o-audio-preview (text+audio, audio) - Azure: gpt-4o-2024-08-06 via Azure OpenAI (text, image, text+image) Output files are saved to: tests/pytest/scripts/provider_stats/ """ import argparse import base64 import json import os import sys import time import uuid from typing import Any def detect_image_media_type(base64_data: str) -> str: """ Detect the actual image format from base64 data by examining the magic bytes. Args: base64_data: Base64-encoded image data. Returns: The detected media type (e.g., 'image/png', 'image/jpeg'). Defaults to 'image/jpeg' if format cannot be determined. """ try: # Decode first few bytes to check magic numbers header = base64.b64decode(base64_data[:32]) # PNG: 89 50 4E 47 0D 0A 1A 0A if header[:8] == b"\x89PNG\r\n\x1a\n": return "image/png" # JPEG: FF D8 FF if header[:3] == b"\xff\xd8\xff": return "image/jpeg" # GIF: GIF87a or GIF89a if header[:6] in (b"GIF87a", b"GIF89a"): return "image/gif" # WebP: RIFF....WEBP if header[:4] == b"RIFF" and header[8:12] == b"WEBP": return "image/webp" except Exception: pass return "image/jpeg" # Add project root to path sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # ============================================================================= # PROVIDER CONFIGURATIONS # ============================================================================= PROVIDER_MODALITY_SUPPORT = { "anthropic": { "model": "claude-sonnet-4-5-20250929", "supported_modalities": ["text-only", "image-only", "text-image"], # Note: Anthropic does not support audio }, "openai": { "model": "gpt-4o-2024-08-06", "supported_modalities": ["text-only", "image-only", "text-image"], }, "openai-audio": { "model": "gpt-4o-audio-preview", "supported_modalities": ["audio-only", "text-audio"], }, "gemini": { "model": "gemini-2.5-flash", "supported_modalities": [ "text-only", "image-only", "audio-only", "text-image", "text-audio", "image-audio", "text-image-audio", ], }, "vertex_ai": { "model": "gemini-2.5-flash", "supported_modalities": [ "text-only", "image-only", "audio-only", "text-image", "text-audio", "image-audio", "text-image-audio", ], }, "azure": { "model": "gpt-4o-2024-08-06", "supported_modalities": ["text-only", "image-only", "text-image"], }, } def load_messages(modality: str, provider: str, messages_dir: str) -> list[dict]: """Load messages from JSON file for a given modality/provider combination.""" filepath = os.path.join(messages_dir, f"{modality}_{provider}.json") with open(filepath) as f: return json.load(f) def transform_messages_for_openai(messages: list[dict]) -> list[dict]: """ Transform palimpzest/litellm message format to OpenAI API format. OpenAI expects: - system messages with string content - user messages with string content or array of content parts Input messages may have content as string or list of content blocks. """ openai_messages = [] for msg in messages: role = msg.get("role") msg_type = msg.get("type") content = msg.get("content") if role == "system": # System content may be string or list of content blocks if isinstance(content, list): # Extract text from content blocks text_parts = [block.get("text", "") for block in content if block.get("type") == "text"] openai_messages.append({"role": "system", "content": "".join(text_parts)}) else: openai_messages.append({"role": "system", "content": content}) elif role == "user": if msg_type == "text": # Content may be string or list of content blocks if isinstance(content, list): # Already content blocks - add them directly content_parts = [] for block in content: # Convert to OpenAI format (remove cache_control if present) openai_block = {"type": block.get("type", "text")} if block.get("type") == "text": openai_block["text"] = block.get("text", "") content_parts.append(openai_block) if openai_messages and openai_messages[-1]["role"] == "user": existing_content = openai_messages[-1]["content"] if isinstance(existing_content, str): openai_messages[-1]["content"] = [ {"type": "text", "text": existing_content} ] + content_parts else: existing_content.extend(content_parts) else: openai_messages.append({"role": "user", "content": content_parts}) else: # String content if openai_messages and openai_messages[-1]["role"] == "user": existing_content = openai_messages[-1]["content"] if isinstance(existing_content, str): openai_messages[-1]["content"] = [ {"type": "text", "text": existing_content}, {"type": "text", "text": content}, ] else: existing_content.append({"type": "text", "text": content}) else: openai_messages.append({"role": "user", "content": content}) elif msg_type == "image": # Image content image_parts = [] for img in content: if img.get("type") == "image_url": image_parts.append(img) if openai_messages and openai_messages[-1]["role"] == "user": existing_content = openai_messages[-1]["content"] if isinstance(existing_content, str): openai_messages[-1]["content"] = [ {"type": "text", "text": existing_content} ] + image_parts else: existing_content.extend(image_parts) else: openai_messages.append({"role": "user", "content": image_parts}) elif msg_type == "input_audio": # Audio content audio_parts = [] for audio in content: if audio.get("type") == "input_audio": audio_parts.append(audio) if openai_messages and openai_messages[-1]["role"] == "user": existing_content = openai_messages[-1]["content"] if isinstance(existing_content, str): openai_messages[-1]["content"] = [ {"type": "text", "text": existing_content} ] + audio_parts else: existing_content.extend(audio_parts) else: openai_messages.append({"role": "user", "content": audio_parts}) return openai_messages def transform_messages_for_anthropic(messages: list[dict]) -> tuple[str | None, list[dict]]: """ Transform palimpzest/litellm message format to Anthropic API format. Input messages may already have cache_control markers from PromptManager. This function preserves those markers while converting to Anthropic's native format. Anthropic expects: - system as a separate parameter (not in messages) - user/assistant messages with content as array of content blocks - cache_control markers for caching (preserved from input) """ system_prompt = None anthropic_messages = [] for msg in messages: role = msg.get("role") msg_type = msg.get("type") content = msg.get("content") if role == "system": # Anthropic uses system as a separate parameter # Content may already be a list of content blocks with cache_control if isinstance(content, list): # Already in content block format (from PromptManager) system_prompt = content else: # String content - wrap in content block with cache_control system_prompt = [ { "type": "text", "text": content, "cache_control": {"type": "ephemeral"}, } ] elif role == "user": if msg_type == "text": # Content may be string or list of content blocks if isinstance(content, list): # Already content blocks (may have cache_control) - preserve them for block in content: if anthropic_messages and anthropic_messages[-1]["role"] == "user": anthropic_messages[-1]["content"].append(block) else: anthropic_messages.append({"role": "user", "content": [block]}) else: # String content content_block = {"type": "text", "text": content} if anthropic_messages and anthropic_messages[-1]["role"] == "user": anthropic_messages[-1]["content"].append(content_block) else: anthropic_messages.append({"role": "user", "content": [content_block]}) elif msg_type == "image": # Image content - Anthropic uses base64 format for img in content: if img.get("type") == "image_url": url = img["image_url"]["url"] if url.startswith("data:"): # Extract base64 data _, data = url.split(";base64,") # Detect actual media type from image data (in case URL has wrong type) media_type = detect_image_media_type(data) image_block = { "type": "image", "source": { "type": "base64", "media_type": media_type, "data": data, }, } # Preserve cache_control if present on the original block if "cache_control" in img: image_block["cache_control"] = img["cache_control"] if anthropic_messages and anthropic_messages[-1]["role"] == "user": anthropic_messages[-1]["content"].append(image_block) else: anthropic_messages.append({"role": "user", "content": [image_block]}) return system_prompt, anthropic_messages def transform_messages_for_gemini(messages: list[dict]) -> tuple[str | None, list[dict]]: """ Transform palimpzest/litellm message format to Gemini API format. Gemini expects: - role: "user" or "model" - parts: list of content parts Input messages may have content as string or list of content blocks. """ gemini_contents = [] system_instruction = None for msg in messages: role = msg.get("role") msg_type = msg.get("type") content = msg.get("content") if role == "system": # Gemini uses system_instruction # Content may be string or list of content blocks if isinstance(content, list): # Extract text from content blocks text_parts = [block.get("text", "") for block in content if block.get("type") == "text"] system_instruction = "".join(text_parts) else: system_instruction = content elif role == "user": parts = [] if msg_type == "text": # Content may be string or list of content blocks if isinstance(content, list): for block in content: if block.get("type") == "text": parts.append({"text": block.get("text", "")}) else: parts.append({"text": content}) elif msg_type == "image": for img in content: if img.get("type") == "image_url": url = img["image_url"]["url"] if url.startswith("data:"): _, data = url.split(";base64,") # Detect actual media type from image data media_type = detect_image_media_type(data) parts.append({ "inline_data": { "mime_type": media_type, "data": data, } }) elif msg_type == "input_audio": for audio in content: if audio.get("type") == "input_audio": audio_data = audio["input_audio"] parts.append({ "inline_data": { "mime_type": f"audio/{audio_data.get('format', 'wav')}", "data": audio_data["data"], } }) if parts: if gemini_contents and gemini_contents[-1]["role"] == "user": gemini_contents[-1]["parts"].extend(parts) else: gemini_contents.append({"role": "user", "parts": parts}) return system_instruction, gemini_contents def call_openai_api(messages: list[dict], model: str, cache_key: str | None = None) -> dict[str, Any]: """ Call OpenAI API directly and return usage statistics. Args: messages: List of message dicts model: Model name cache_key: Optional prompt_cache_key for sticky routing to same cache shard Returns dict with: - completion_tokens - prompt_tokens - prompt_tokens_details (cached_tokens, text_tokens, image_tokens, audio_tokens) - total_tokens """ import openai client = openai.OpenAI() openai_messages = transform_messages_for_openai(messages) kwargs = {"model": model, "messages": openai_messages, "temperature": 0.0} # Check if this is an audio model if "audio" in model: kwargs["modalities"] = ["text"] # Add prompt_cache_key for caching (ensures requests route to same cache shard) if cache_key: kwargs["extra_body"] = {"prompt_cache_key": cache_key} response = client.chat.completions.create(**kwargs) # Extract complete usage stats usage_dict = {} if response.usage: usage_dict = response.usage.model_dump() # Get response text safely try: response_text = response.choices[0].message.content[:200] if response.choices and response.choices[0].message.content else None except Exception: response_text = None # Serialize the full response try: raw_response = response.model_dump() except Exception: raw_response = str(response) return { "provider": "openai", "model": model, "usage": usage_dict, "response_content": response_text, "raw_response": raw_response, } # NOTE: this function was generated speculatively and has not been tested, so it may have errors def call_azure_api(messages: list[dict], model: str, cache_key: str | None = None) -> dict[str, Any]: """ Call Azure OpenAI API directly and return usage statistics. Uses the same message format as OpenAI, but routes through Azure endpoints. Args: messages: List of message dicts model: Model name (deployment name) cache_key: Optional prompt_cache_key for sticky routing to same cache shard Returns dict with: - completion_tokens - prompt_tokens - prompt_tokens_details (cached_tokens, text_tokens, image_tokens, audio_tokens) - total_tokens """ import openai api_key = os.environ.get("AZURE_API_KEY") or os.environ.get("AZURE_OPENAI_API_KEY") azure_endpoint = os.environ.get("AZURE_API_BASE") api_version = os.environ.get("AZURE_API_VERSION", "2024-12-01-preview") if not api_key: raise ValueError("AZURE_API_KEY or AZURE_OPENAI_API_KEY must be set") if not azure_endpoint: raise ValueError("AZURE_API_BASE must be set") client = openai.AzureOpenAI( api_key=api_key, azure_endpoint=azure_endpoint, api_version=api_version, ) openai_messages = transform_messages_for_openai(messages) kwargs = {"model": model, "messages": openai_messages, "temperature": 0.0} # Add prompt_cache_key for caching (ensures requests route to same cache shard) if cache_key: kwargs["extra_body"] = {"prompt_cache_key": cache_key} response = client.chat.completions.create(**kwargs) # Extract complete usage stats usage_dict = {} if response.usage: usage_dict = response.usage.model_dump() # Get response text safely try: response_text = response.choices[0].message.content[:200] if response.choices and response.choices[0].message.content else None except Exception: response_text = None # Serialize the full response try: raw_response = response.model_dump() except Exception: raw_response = str(response) return { "provider": "azure", "model": model, "usage": usage_dict, "response_content": response_text, "raw_response": raw_response, } def call_anthropic_api(messages: list[dict], model: str) -> dict[str, Any]: """ Call Anthropic API directly and return usage statistics. Returns dict with: - input_tokens - output_tokens - cache_creation_input_tokens - cache_read_input_tokens """ import anthropic client = anthropic.Anthropic() system_prompt, anthropic_messages = transform_messages_for_anthropic(messages) response = client.messages.create( model=model, max_tokens=1024, system=system_prompt, messages=anthropic_messages, ) # Extract complete usage stats usage_dict = {} if response.usage: usage_dict = response.usage.model_dump() # Get response text safely try: response_text = response.content[0].text[:200] if response.content and response.content[0].text else None except Exception: response_text = None # Serialize the full response try: raw_response = response.model_dump() except Exception: raw_response = str(response) return { "provider": "anthropic", "model": model, "usage": usage_dict, "response_content": response_text, "raw_response": raw_response, } def call_gemini_api(messages: list[dict], model: str, use_vertex: bool = False) -> dict[str, Any]: """ Call Gemini API directly and return usage statistics. Args: messages: List of message dicts model: Model name use_vertex: If True, use Vertex AI; otherwise use Google AI Studio Returns dict with usage statistics. """ from google import genai from google.genai import types system_instruction, gemini_contents = transform_messages_for_gemini(messages) # Create client for Google AI Studio or Vertex AI if use_vertex: # Vertex AI requires project and location import os client = genai.Client( vertexai=True, project=os.environ.get("GOOGLE_CLOUD_PROJECT", os.environ.get("VERTEXAI_PROJECT")), location=os.environ.get("GOOGLE_CLOUD_LOCATION", os.environ.get("VERTEXAI_LOCATION", "us-central1")), ) else: # Google AI Studio uses API key from environment client = genai.Client() # Build the config config = types.GenerateContentConfig( temperature=0.0, system_instruction=system_instruction if system_instruction else None, ) response = client.models.generate_content( model=model, contents=gemini_contents, config=config, ) # Extract complete usage stats from usage_metadata usage_metadata = response.usage_metadata usage_dict = {} if usage_metadata: # Try model_dump() first (Pydantic models), then to_dict(), then manual extraction try: usage_dict = usage_metadata.model_dump() except AttributeError: try: usage_dict = usage_metadata.to_dict() except AttributeError: # Manual extraction of known Gemini usage fields usage_dict = { "prompt_token_count": getattr(usage_metadata, "prompt_token_count", None), "candidates_token_count": getattr(usage_metadata, "candidates_token_count", None), "total_token_count": getattr(usage_metadata, "total_token_count", None), "cached_content_token_count": getattr(usage_metadata, "cached_content_token_count", None), } # Get response text safely try: response_text = response.text[:200] if response.text else None except Exception: response_text = None # Serialize the full response try: # Try model_dump() first (Pydantic models) raw_response = response.model_dump() except AttributeError: try: raw_response = response.to_dict() except AttributeError: # Manual serialization try: raw_response = { "text": response.text if hasattr(response, "text") else None, "candidates": [ { "content": { "parts": [{"text": getattr(part, "text", str(part))} for part in c.content.parts] if c.content and c.content.parts else [], "role": c.content.role if c.content else None, }, "finish_reason": str(c.finish_reason) if hasattr(c, "finish_reason") else None, } for c in (response.candidates or []) ], "usage_metadata": usage_dict, "model_version": getattr(response, "model_version", None), } except Exception as e: raw_response = {"error": str(e), "response_str": str(response)} return { "provider": "vertex_ai" if use_vertex else "gemini", "model": model, "usage": usage_dict, "response_content": response_text, "raw_response": raw_response, } def capture_stats_for_provider( provider: str, modality: str, messages: list[dict], model: str, ) -> dict[str, Any]: """ Capture stats for a provider by making two requests with a 10-second gap. Returns dict with: - first_request: stats from first request - second_request: stats from second request (should show cache hits) """ # Generate a unique cache key for OpenAI/Azure (ensures both requests hit the same cache shard) openai_cache_key = f"pz-test-{uuid.uuid4().hex[:12]}" if provider in ("openai", "openai-audio", "azure") else None print(" First request...") if provider == "openai" or provider == "openai-audio": first_stats = call_openai_api(messages, model, cache_key=openai_cache_key) elif provider == "azure": first_stats = call_azure_api(messages, model, cache_key=openai_cache_key) elif provider == "anthropic": first_stats = call_anthropic_api(messages, model) elif provider == "gemini": first_stats = call_gemini_api(messages, model, use_vertex=False) elif provider == "vertex_ai": first_stats = call_gemini_api(messages, model, use_vertex=True) else: raise ValueError(f"Unknown provider: {provider}") print(f" Usage: {first_stats['usage']}") print(" Waiting 20 seconds for cache to be available...") time.sleep(20) print(" Second request (should show cache hits)...") if provider == "openai" or provider == "openai-audio": second_stats = call_openai_api(messages, model, cache_key=openai_cache_key) elif provider == "azure": second_stats = call_azure_api(messages, model, cache_key=openai_cache_key) elif provider == "anthropic": second_stats = call_anthropic_api(messages, model) elif provider == "gemini": second_stats = call_gemini_api(messages, model, use_vertex=False) elif provider == "vertex_ai": second_stats = call_gemini_api(messages, model, use_vertex=True) print(f" Usage: {second_stats['usage']}") return { "provider": provider, "model": model, "modality": modality, "first_request": first_stats, "second_request": second_stats, } def save_stats(stats: dict[str, Any], output_dir: str, provider: str, modality: str) -> str: """Save stats to a JSON file.""" os.makedirs(output_dir, exist_ok=True) output_path = os.path.join(output_dir, f"{provider}_{modality}.json") with open(output_path, "w") as f: json.dump(stats, f, indent=2) return output_path def main(): """Capture provider stats for supported provider/modality combinations.""" parser = argparse.ArgumentParser( description="Capture token/cost statistics from LLM providers.", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=f""" Available providers: {', '.join(PROVIDER_MODALITY_SUPPORT.keys())} Available modalities: text-only, image-only, audio-only, text-image, text-audio, image-audio, text-image-audio Examples: python capture_provider_stats.py # Run all providers/modalities python capture_provider_stats.py -p openai # Run all modalities for OpenAI python capture_provider_stats.py -p openai -m text-only # Run only text-only for OpenAI python capture_provider_stats.py -p anthropic -m text-image # Run only text-image for Anthropic """ ) parser.add_argument( "-p", "--provider", nargs="+", choices=list(PROVIDER_MODALITY_SUPPORT.keys()), help="Provider(s) to run. If not specified, runs all providers.", ) parser.add_argument( "-m", "--modality", nargs="+", choices=["text-only", "image-only", "audio-only", "text-image", "text-audio", "image-audio", "text-image-audio"], help="Modality(ies) to run. If not specified, runs all supported modalities for each provider.", ) args = parser.parse_args() messages_dir = os.path.join( os.path.dirname(__file__), "..", "tests", "pytest", "data", "generator_messages", ) messages_dir = os.path.abspath(messages_dir) output_dir = os.path.join( os.path.dirname(__file__), "provider_stats", ) output_dir = os.path.abspath(output_dir) print(f"Loading messages from: {messages_dir}") print(f"Saving stats to: {output_dir}\n") # Determine which providers to run providers_to_run = args.provider if args.provider else list(PROVIDER_MODALITY_SUPPORT.keys()) print(f"Providers to run: {providers_to_run}\n") for provider in providers_to_run: config = PROVIDER_MODALITY_SUPPORT[provider] model = config["model"] supported_modalities = config["supported_modalities"] # Filter modalities if specified if args.modality: modalities_to_run = [m for m in args.modality if m in supported_modalities] if not modalities_to_run: print(f"\nProvider: {provider} - SKIPPED (none of {args.modality} supported)") continue else: modalities_to_run = supported_modalities print(f"\nProvider: {provider} (model: {model})") print(f" Modalities to run: {modalities_to_run}") for modality in modalities_to_run: print(f"\n Processing modality: {modality}") try: messages = load_messages(modality, provider, messages_dir) print(f" Loaded {len(messages)} messages from {modality}_{provider}.json") stats = capture_stats_for_provider(provider, modality, messages, model) output_path = save_stats(stats, output_dir, provider, modality) print(f" Saved to: {output_path}") except FileNotFoundError as e: print(f" SKIPPED: Message file not found - {e}") except Exception as e: print(f" ERROR: {e}") import traceback traceback.print_exc() print("\nDone!") if __name__ == "__main__": main() ================================================ FILE: scripts/generate_test_messages.py ================================================ #!/usr/bin/env python3 """ Script to generate test messages for each provider/modality combination. This script uses the Generator class directly to create message payloads. It uses the 'generating_messages_only' flag to retrieve the exact messages that would be sent to the provider without making an actual API call. Supported provider/modality combinations: - Anthropic: text-only, image-only, text-image (no audio support) - OpenAI: text-only, image-only, text-image - OpenAI-Audio: audio-only, text-audio - Gemini: all 7 modality combinations - Vertex AI: all 7 modality combinations - Azure: text-only, image-only, text-image Output files are saved to: tests/pytest/data/generator_messages/ Format: {modality}_{provider}.json (e.g., text-only_anthropic.json) """ import json import os import sys from pydantic import BaseModel, Field # Add project root to path sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from palimpzest.constants import Model, PromptStrategy from palimpzest.core.elements.records import DataRecord from palimpzest.core.lib.schemas import AudioFilepath, ImageFilepath, union_schemas from palimpzest.query.generators.generators import Generator def generate_session_id(provider: str, modality: str) -> str: """ Generate a unique 12-character session ID for a provider/modality combination. This ensures each modality test has a unique prompt prefix, preventing cross-modality cache hits. The ID is deterministic based on provider+modality so regenerating produces consistent results. """ import hashlib hash_input = f"{provider}_{modality}" hash_hex = hashlib.md5(hash_input.encode()).hexdigest() return hash_hex[:12].upper() STATIC_CONTEXT = """ WILDLIFE CONSERVATION & RESEARCH CENTER: SPECIES IDENTIFICATION MANUAL (v2025.1) SECTION 1: INTRODUCTION AND MISSION The Wildlife Conservation & Research Center (WCRC) is dedicated to the preservation, study, and rehabilitation of diverse wildlife species. All staff members, researchers, and volunteers must adhere to these protocols for accurate species identification and data collection. Our mission combines advanced biological sciences with conservation efforts to protect endangered and threatened populations worldwide. SECTION 2: MAMMAL IDENTIFICATION PROTOCOLS 2.1 ELEPHANTS (Family Elephantidae): - African Savanna Elephant: Larger ears (shaped like Africa), concave back, two fingers on trunk tip. Weight: 5,000-14,000 lbs. - African Forest Elephant: Smaller stature, oval-shaped ears, straighter tusks pointing downward. - Asian Elephant: Smaller ears, convex back, one finger on trunk tip, twin domes on head. Weight: 4,000-11,000 lbs. - Vocalizations: Trumpeting (alarm/excitement), rumbling (long-distance communication), roaring (distress). 2.2 BIG CATS (Family Felidae): - Lion (Panthera leo): Tawny coat, males have distinctive mane. Social, live in prides. Height: 3.5-4 ft at shoulder. - Tiger (Panthera tigris): Orange coat with black stripes, white underbelly. Solitary hunters. Largest cat species. - Leopard (Panthera pardus): Golden-yellow coat with rosette patterns. Excellent climbers, often cache prey in trees. - Cheetah (Acinonyx jubatus): Spotted coat, black "tear marks" from eyes to mouth. Fastest land animal (70 mph). - Vocalizations: Roaring (lions, tigers, leopards), chirping/purring (cheetahs cannot roar). 2.3 BEARS (Family Ursidae): - Brown Bear (Ursus arctos): Large shoulder hump, dish-shaped face, long claws. Includes grizzly subspecies. - Black Bear (Ursus americanus): Straight facial profile, no shoulder hump, shorter claws. Most common North American bear. - Polar Bear (Ursus maritimus): White fur, longer neck, smaller ears. Marine mammal adapted to Arctic conditions. - Giant Panda (Ailuropoda melanoleuca): Black and white coloring, feeds almost exclusively on bamboo. - Vocalizations: Roaring, growling, huffing, jaw-popping (threat displays). 2.4 PRIMATES (Order Primates): - Gorilla: Largest primate, silver-back males, knuckle-walking locomotion. Vocalizations include chest-beating, hooting. - Chimpanzee: Highly intelligent, uses tools, complex social structures. Vocalizations: pant-hoots, screams. - Orangutan: Red-orange fur, arboreal lifestyle, solitary. Long calls can travel over 1 km. - Gibbon: Smaller apes, brachiation locomotion, distinctive whooping songs for territorial marking. SECTION 3: BIRD IDENTIFICATION PROTOCOLS 3.1 RAPTORS (Order Accipitriformes/Falconiformes): - Bald Eagle: White head and tail, yellow beak. Wingspan: 6-7.5 ft. Call: high-pitched chattering. - Golden Eagle: Dark brown plumage, golden nape. Powerful hunters of small mammals. - Peregrine Falcon: Blue-gray back, barred underparts. Fastest bird in dive (240+ mph). - Red-tailed Hawk: Brown back, pale underparts, distinctive red tail. Most common North American hawk. 3.2 PARROTS (Order Psittaciformes): - Macaw: Large, colorful, long tail feathers. Powerful curved beaks. Highly social and vocal. - African Grey: Gray plumage, red tail. Exceptional mimicry and cognitive abilities. - Cockatoo: White or pink plumage, distinctive crest. Loud screeching vocalizations. SECTION 4: REPTILE IDENTIFICATION PROTOCOLS 4.1 CROCODILIANS (Order Crocodilia): - American Alligator: Broad, U-shaped snout, dark coloration. Freshwater habitats. - Nile Crocodile: V-shaped snout, aggressive. Can reach 16-18 ft in length. - Gharial: Extremely narrow snout, fish-eating specialist. Critically endangered. 4.2 LARGE SNAKES (Families Pythonidae/Boidae): - Reticulated Python: Longest snake species (up to 23 ft), complex geometric patterns. - Green Anaconda: Heaviest snake species, olive-green with black spots. Semi-aquatic. - King Cobra: Longest venomous snake (up to 18 ft), distinctive hood when threatened. SECTION 5: DATA COLLECTION AND ANALYSIS 5.1 Visual Identification: - Document body shape, size, coloration, and distinctive markings. - Note behavioral characteristics and habitat context. - Use standardized photography protocols for pattern matching. 5.2 Audio Identification: - Record vocalizations with frequency analysis equipment. - Tag recordings with behavioral context (territorial, mating, alarm, social). - Cross-reference with vocalization databases for species confirmation. 5.3 Biometric Data: - Record body measurements according to species-specific protocols. - Document age indicators (teeth wear, plumage, etc.). - Collect genetic samples when possible for lineage verification. You are an AI Research Assistant for the WCRC. Your job is to analyze data inputs (text descriptions, images, and/or audio recordings) and identify the species based on the characteristics described in this manual. Analyze all provided inputs and determine the most likely species identification. """ class TextInputSchema(BaseModel): """Schema for text-only input.""" text: str = Field(description="Description of an animal") age: int = Field(description="The age of the animal in years") class ImageInputSchema(BaseModel): """Schema for image-only input.""" image_file: ImageFilepath = Field(description="File path to an image of an animal") height: float = Field(description="The estimated height of the animal in cm") class AudioInputSchema(BaseModel): """Schema for audio-only input.""" audio_file: AudioFilepath = Field(description="File path to an audio recording of an animal") year: float = Field(description="The year the recording was made") # Union schemas for multi-modal inputs TextImageInputSchema = union_schemas([TextInputSchema, ImageInputSchema]) TextAudioInputSchema = union_schemas([TextInputSchema, AudioInputSchema]) ImageAudioInputSchema = union_schemas([ImageInputSchema, AudioInputSchema]) TextImageAudioInputSchema = union_schemas([TextInputSchema, ImageInputSchema, AudioInputSchema]) class OutputSchema(BaseModel): """Output schema for animal identification.""" animal: str = Field(description="The animal in the input") MODALITY_CONFIGS = { "text-only": { "input_schema": TextInputSchema, "data_item": { "text": "An elephant is a large gray animal with a trunk and big ears. It makes a trumpeting sound.", "age": 15, }, }, "image-only": { "input_schema": ImageInputSchema, "data_item": { "image_file": "tests/pytest/data/elephant.png", "height": 304.5, }, }, "audio-only": { "input_schema": AudioInputSchema, "data_item": { "audio_file": "tests/pytest/data/elephant.wav", "year": 2020, }, }, "text-image": { "input_schema": TextImageInputSchema, "data_item": { "text": "An elephant is a large gray animal with a trunk and big ears. It makes a trumpeting sound.", "age": 15, "image_file": "tests/pytest/data/elephant.png", "height": 304.5, }, }, "text-audio": { "input_schema": TextAudioInputSchema, "data_item": { "text": "An elephant is a large gray animal with a trunk and big ears. It makes a trumpeting sound.", "age": 15, "audio_file": "tests/pytest/data/elephant.wav", "year": 2020, }, }, "image-audio": { "input_schema": ImageAudioInputSchema, "data_item": { "image_file": "tests/pytest/data/elephant.png", "height": 304.5, "audio_file": "tests/pytest/data/elephant.wav", "year": 2020, }, }, "text-image-audio": { "input_schema": TextImageAudioInputSchema, "data_item": { "text": "An elephant is a large gray animal with a trunk and big ears. It makes a trumpeting sound.", "age": 15, "image_file": "tests/pytest/data/elephant.png", "height": 304.5, "audio_file": "tests/pytest/data/elephant.wav", "year": 2020, }, }, } # Maps provider name to (Model enum, supported modalities) PROVIDER_CONFIGS = { "anthropic": { "model": Model.CLAUDE_4_5_SONNET, "supported_modalities": ["text-only", "image-only", "text-image"], }, "openai": { "model": Model.GPT_4o, "supported_modalities": ["text-only", "image-only", "text-image"], }, "openai-audio": { "model": Model.GPT_4o_AUDIO_PREVIEW, "supported_modalities": ["audio-only", "text-audio"], }, "gemini": { "model": Model.GOOGLE_GEMINI_2_5_FLASH, "supported_modalities": [ "text-only", "image-only", "audio-only", "text-image", "text-audio", "image-audio", "text-image-audio", ], }, "vertex_ai": { "model": Model.GEMINI_2_5_FLASH, "supported_modalities": [ "text-only", "image-only", "audio-only", "text-image", "text-audio", "image-audio", "text-image-audio", ], }, "azure": { "model": Model.AZURE_GPT_4o, "supported_modalities": ["text-only", "image-only", "text-image"], }, } def save_messages(modality: str, provider: str, messages: list[dict], output_dir: str) -> str: """ Save messages to a JSON file. Args: modality: Modality name provider: Provider name messages: List of message dicts output_dir: Directory to save files Returns: Path to the saved file """ os.makedirs(output_dir, exist_ok=True) output_path = os.path.join(output_dir, f"{modality}_{provider}.json") # Convert messages to JSON-serializable format serializable_messages = [] for msg in messages: serializable_msg = msg.copy() serializable_messages.append(serializable_msg) with open(output_path, "w") as f: json.dump(serializable_messages, f, indent=2, default=str) return output_path def main(): """Generate and save messages for all provider/modality combinations.""" # Ensure the output directory follows the repository structure output_dir = os.path.join( os.path.dirname(__file__), "..", "tests", "pytest", "data", "generator_messages", ) output_dir = os.path.abspath(output_dir) # Count total combinations total_combinations = sum( len(provider_config["supported_modalities"]) for provider_config in PROVIDER_CONFIGS.values() ) print(f"Generating test messages for {total_combinations} provider/modality combinations...") print(f"Output directory: {output_dir}") print(f"Static context length: ~{len(STATIC_CONTEXT.split())} words\n") generated_count = 0 for provider, provider_config in PROVIDER_CONFIGS.items(): model = provider_config["model"] supported_modalities = provider_config["supported_modalities"] print(f"Provider: {provider} (model: {model.value})") print(f" Supported modalities: {supported_modalities}") for modality in supported_modalities: config = MODALITY_CONFIGS[modality] print(f" Generating: {modality}_{provider}") try: # Prepare input record input_schema = config["input_schema"] data_item = config["data_item"] input_record = DataRecord(input_schema(**data_item), source_indices=[0]) # Instantiate Generator generator = Generator( model=model, prompt_strategy=PromptStrategy.MAP, reasoning_effort=None, desc=STATIC_CONTEXT, ) # Generate unique session ID for this provider/modality to prevent cross-modality cache hits session_id = generate_session_id(provider, modality) # Call the generator with the new flag # Pass cache_isolation_id to inject session ID at start of system/user prompts messages = generator( candidate=input_record, fields=OutputSchema.model_fields, output_schema=OutputSchema, generating_messages_only=True, cache_isolation_id=session_id, ) # Manually save the messages using local helper output_path = save_messages(modality, provider, messages, output_dir) print(f" Session ID: {session_id}") print(f" Saved to: {output_path}") print(f" Messages: {len(messages)}") # Print message summary for i, msg in enumerate(messages): role = msg.get("role", "unknown") msg_type = msg.get("type", "unknown") content = msg.get("content", "") content_len = len(content) if isinstance(content, str) else len(str(content)) print(f" [{i}] role={role}, type={msg_type}, len={content_len}") generated_count += 1 except Exception as e: print(f" ERROR: {e}") import traceback traceback.print_exc() print() print(f"Done! Generated {generated_count}/{total_combinations} message files.") if __name__ == "__main__": main() ================================================ FILE: scripts/update_model_info.py ================================================ #!/usr/bin/env python3 """ Script to automatically update pz_models_information.json with data from external sources. Data Sources: - LiteLLM proxy /model/info endpoint: Dynamic model info (100% accuracy, prioritized) - LiteLLM model_prices_and_context_window.json: Cost and capability data (fallback) - MMLU-Pro leaderboard: Quality scores (fuzzy matching acceptable) - Artificial Analysis: Latency data (fuzzy matching acceptable) Usage: python scripts/update_model_info.py MODEL_ID [MODEL_ID ...] [--use-endpoint] """ import argparse import json import os import socket import subprocess import sys import time from typing import Any import requests import yaml # Add src to path to import from palimpzest sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) from palimpzest.utils.model_info_helpers import ( LATENCY_TPS_DATA, MMLU_PRO_SCORES, derive_model_flags, fuzzy_match_score, ) # Constants LITELLM_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" PZ_MODELS_PATH = os.path.join( os.path.dirname(__file__), "..", "src", "palimpzest", "utils", "pz_models_information.json", ) # Provider mapping from LiteLLM prefixes to our provider strings PROVIDER_MAPPING = { "openai": "openai", "anthropic": "anthropic", "claude": "anthropic", "vertex_ai": "vertex_ai", "gemini": "gemini", "together_ai": "together_ai", "together": "together_ai", "hosted_vllm": "hosted_vllm", "groq": "groq", "mistral": "mistral", "cohere": "cohere", "bedrock": "bedrock", "azure": "azure", "deepseek": "deepseek", "fireworks_ai": "fireworks_ai", "xai": "xai", } # API key environment variable mapping API_KEY_MAPPING = { "openai": "OPENAI_API_KEY", "azure": "AZURE_API_KEY", "anthropic": "ANTHROPIC_API_KEY", "vertex_ai": "GOOGLE_APPLICATION_CREDENTIALS", "gemini": "GEMINI_API_KEY", "together_ai": "TOGETHER_API_KEY", "hosted_vllm": "VLLM_API_KEY", "groq": "GROQ_API_KEY", "mistral": "MISTRAL_API_KEY", "cohere": "COHERE_API_KEY", "deepseek": "DEEPSEEK_API_KEY", "fireworks_ai": "FIREWORKS_API_KEY", "xai": "XAI_API_KEY", } # Field mapping from LiteLLM endpoint to PZ format FIELD_MAPPING = [ ("usd_per_input_token", "input_cost_per_token", None), ("usd_per_output_token", "output_cost_per_token", None), ("usd_per_audio_input_token", "input_cost_per_audio_token", None), ("usd_per_audio_output_token", "output_cost_per_audio_token", None), ("usd_per_image_output_token", "output_cost_per_image_token", None), ("usd_per_cache_read_token", "cache_read_input_token_cost", None), ("usd_per_cache_creation_token", "cache_creation_input_token_cost", None), ("supports_prompt_caching", "supports_prompt_caching", False), ] # Boolean capability fields derived from endpoint CAPABILITY_MAPPING = [ ("is_vision_model", "supports_vision", False), ("is_audio_model", "supports_audio_input", False), ("is_reasoning_model", "supports_reasoning", False), ] # MMLU_PRO_SCORES, LATENCY_TPS_DATA, and fuzzy_match_score are imported from # palimpzest.utils.model_info_helpers # Alias for backwards compatibility in this script LATENCY_DATA = LATENCY_TPS_DATA # ============================================================================= # LiteLLM Proxy Endpoint Functions # ============================================================================= def get_free_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(("", 0)) return s.getsockname()[1] def extract_provider(model_id: str) -> str: """Extract provider from model ID.""" if "/" in model_id: prefix = model_id.split("/")[0].lower() return PROVIDER_MAPPING.get(prefix, prefix) model_lower = model_id.lower() # OpenAI if any(x in model_lower for x in ["gpt", "o1-", "o3-", "o4-", "dall-e", "whisper"]): return "openai" # Anthropic if "claude" in model_lower: return "anthropic" # Google (Vertex AI / Gemini) if "gemini" in model_lower or "bison" in model_lower: return "vertex_ai" # Meta / Together / Llama if "llama" in model_lower: return "together_ai" # Mistral if "mistral" in model_lower or "mixtral" in model_lower: return "mistral" # DeepSeek if "deepseek" in model_lower: return "deepseek" return "unknown" def get_api_key_env_var(provider: str) -> str | None: return API_KEY_MAPPING.get(provider) def generate_config_yaml(model_ids: list[str]) -> str: config_id = 0 config_filename = f"litellm_config_{config_id}.yaml" while not os.path.exists(config_filename): config_id += 1 config_list = [] for model_id in model_ids: provider = extract_provider(model_id) env_var_name = get_api_key_env_var(provider) api_key_val = f"os.environ/{env_var_name}" if env_var_name else None entry = { "model_name": model_id, "litellm_params": { "model": model_id, "api_key": api_key_val, }, } config_list.append(entry) yaml_structure = {"model_list": config_list} with open(config_filename, "w") as f: yaml.dump(yaml_structure, f, default_flow_style=False, sort_keys=False) return config_filename def fetch_dynamic_model_info(model_ids: list[str]) -> dict[str, Any]: if not model_ids: return {} port = get_free_port() proxy_url = f"http://127.0.0.1:{port}" config_filename = generate_config_yaml(model_ids) server_env = os.environ.copy() process = None dynamic_model_info = {} print(f"Starting LiteLLM proxy on port {port} for {len(model_ids)} models...") try: process = subprocess.Popen( ["litellm", "--config", config_filename, "--port", str(port)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=server_env, ) server_ready = False max_retries = 30 for i in range(max_retries): if process.poll() is not None: _, stderr = process.communicate() print(f" LiteLLM process died unexpectedly: {stderr.decode()}") break try: requests.get(f"{proxy_url}/health/readiness", timeout=1) server_ready = True print(f" Server ready after {i + 1} attempts") break except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): time.sleep(0.5) if not server_ready: print(" Timeout: LiteLLM server failed to start within the limit.") return {} try: response = requests.get(f"{proxy_url}/model/info", timeout=10) response.raise_for_status() model_data = response.json() if "data" in model_data and len(model_data["data"]) > 0: for item in model_data["data"]: model_name = item.get("model_name") model_info = item.get("model_info", {}) dynamic_model_info[model_name] = model_info print(f" Retrieved info for: {model_name}") else: print(" WARNING: No model data returned from endpoint") except Exception as e: print(f" Error fetching model info: {e}") finally: if process: process.terminate() try: process.wait(timeout=5) except subprocess.TimeoutExpired: process.kill() if os.path.exists(config_filename): os.remove(config_filename) return dynamic_model_info # ============================================================================= # Data Fetching Functions # ============================================================================= def fetch_litellm_data() -> dict[str, Any]: print(f"Fetching LiteLLM data from {LITELLM_URL}...") try: response = requests.get(LITELLM_URL, timeout=30) response.raise_for_status() data = response.json() print(f" Found {len(data)} models in LiteLLM database") return data except Exception as e: print(f" Error fetching LiteLLM data: {e}") return {} def load_existing_data() -> dict[str, Any]: if os.path.exists(PZ_MODELS_PATH): with open(PZ_MODELS_PATH) as f: return json.load(f) return {} def save_data(data: dict[str, Any]) -> None: with open(PZ_MODELS_PATH, "w") as f: json.dump(data, f, indent=4) print(f" [System] Successfully saved to {PZ_MODELS_PATH}") # ============================================================================= # Matching and Conversion Functions # ============================================================================= # fuzzy_match_score is imported from palimpzest.utils.model_info_helpers def derive_model_flags_with_provider(model_id: str, provider: str) -> dict[str, bool]: """Wrapper around derive_model_flags that also adds provider-specific flags.""" flags = derive_model_flags(model_id) if provider == "hosted_vllm": flags["is_vllm_model"] = True return flags # ============================================================================= # Interactive Review Functions # ============================================================================= def prompt_for_value(field_name: str, current_value: Any, value_type: str = "any") -> Any: while True: user_input = input(f" > Enter new value for '{field_name}' (or press Enter to keep current): ").strip() if user_input == "": return current_value try: if user_input.lower() == "none": return None if value_type == "float": return float(user_input) elif value_type == "int": return int(user_input) elif value_type == "bool": return user_input.lower() in ("true", "yes", "1", "y") else: try: return json.loads(user_input) except json.JSONDecodeError: return user_input except ValueError as e: print(f" Invalid input: {e}. Try again.") def review_field( field_name: str, value: Any, from_endpoint: bool, interactive: bool = True, value_type: str = "any" ) -> tuple[Any, bool]: """ Review a single field. Logic: 1. If from_endpoint is True and value not None -> VERIFIED (return immediately) 2. If interactive -> Ask User (1. Correct, 2. Incorrect) """ if from_endpoint and value is not None: # Verified automatically by endpoint return value, False if not interactive: return value, False print(f"\n [Review] {field_name}: {value}") if from_endpoint and value is None: print(" (Source: Endpoint returned Null)") else: print(" (Source: Derived/Static/Fallback)") while True: choice = input(" 1. Yes, information is correct\n 2. No, enter different value\n Choice [1]: ").strip() if choice == "" or choice == "1": return value, False elif choice == "2": new_value = prompt_for_value(field_name, value, value_type) return new_value, True else: print(" Invalid choice.") def convert_and_review_model( model_id: str, litellm_static: dict[str, Any] | None, litellm_dynamic: dict[str, Any] | None, existing_entry: dict[str, Any] | None, interactive: bool = True, ) -> dict[str, Any]: """ 1. Aggregates all data into a Draft Entry. 2. Displays the Draft Entry (User can see Current State). 3. Iterates fields to Verify (Prioritizing endpoint). """ print(f"\n{'='*60}") print(f"PROCESSING: {model_id}") print(f"{'='*60}") # --- PHASE 1: Build Draft Entry & Source Map --- endpoint_fields: set[str] = set() raw_data: dict[str, Any] = {} # 1. Base: Static Data if litellm_static: raw_data.update(litellm_static) # 2. Overlay: Dynamic Data (Priority) if litellm_dynamic: for key, val in litellm_dynamic.items(): if val is not None: raw_data[key] = val endpoint_fields.add(key) # 3. Construct Candidate dictionary candidate = {} source_map = {} # Map field -> is_from_endpoint # Provider prov = raw_data.get("litellm_provider") or extract_provider(model_id) candidate["provider"] = prov source_map["provider"] = "litellm_provider" in endpoint_fields # Costs & Caching for pz_field, litellm_field, default in FIELD_MAPPING: val = raw_data.get(litellm_field, default) candidate[pz_field] = val source_map[pz_field] = litellm_field in endpoint_fields # Capabilities for pz_field, litellm_field, default in CAPABILITY_MAPPING: val = raw_data.get(litellm_field, default) # Special logic for audio if pz_field == "is_audio_model": audio_in = raw_data.get("supports_audio_input", False) audio_out = raw_data.get("supports_audio_output", False) val = audio_in or audio_out source_map[pz_field] = ("supports_audio_input" in endpoint_fields or "supports_audio_output" in endpoint_fields) else: source_map[pz_field] = litellm_field in endpoint_fields candidate[pz_field] = val # Modes mode = raw_data.get("mode", "chat") mode_src = "mode" in endpoint_fields candidate["is_text_model"] = mode in ["chat", "completion"] source_map["is_text_model"] = mode_src candidate["is_embedding_model"] = mode == "embedding" source_map["is_embedding_model"] = mode_src # Flags (Always derived, never endpoint) flags = derive_model_flags_with_provider(model_id, candidate["provider"]) for k, v in flags.items(): candidate[k] = v source_map[k] = False # Scores / Latency (Fuzzy or Existing) mmlu = fuzzy_match_score(model_id, MMLU_PRO_SCORES) if mmlu is None and existing_entry: mmlu = existing_entry.get("MMLU_Pro_score") candidate["MMLU_Pro_score"] = mmlu source_map["MMLU_Pro_score"] = False tps = fuzzy_match_score(model_id, LATENCY_DATA) sec_per_tok = round(1.0 / tps, 6) if tps else None if sec_per_tok is None and existing_entry: sec_per_tok = existing_entry.get("seconds_per_output_token") candidate["seconds_per_output_token"] = sec_per_tok source_map["seconds_per_output_token"] = False # Audio Cache Read (check existing) acr = existing_entry.get("usd_per_audio_cache_read_token") if existing_entry else None if acr is not None: candidate["usd_per_audio_cache_read_token"] = acr source_map["usd_per_audio_cache_read_token"] = False # Note if existing_entry and existing_entry.get("note"): candidate["note"] = existing_entry["note"] source_map["note"] = False # Sources src_list = [LITELLM_URL] if existing_entry and existing_entry.get("sources"): existing_srcs = existing_entry["sources"] if isinstance(existing_srcs, list): src_list = list(set(src_list + existing_srcs)) elif existing_srcs: src_list = list(set(src_list + [existing_srcs])) candidate["sources"] = src_list # --- PHASE 2: Display Current State --- print("\n--- Current State (Draft) ---") display_dict = {} for k, v in candidate.items(): if k == "sources": continue src_label = "ENDPOINT" if source_map.get(k, False) and v is not None else "DERIVED/STATIC" display_dict[k] = f"{v} [{src_label}]" print(json.dumps(display_dict, indent=2)) print("-" * 30) # --- PHASE 3: Verification Loop --- final_entry = {} final_entry["sources"] = candidate["sources"] # Iterate over specific keys to ensure order and types # Provider final_entry["provider"], _ = review_field( "provider", candidate["provider"], source_map["provider"], interactive, "str" ) # All cost/cache fields for k in [f[0] for f in FIELD_MAPPING] + ["usd_per_audio_cache_read_token"]: if k in candidate: vtype = "float" if "usd_" in k else "bool" final_entry[k], _ = review_field( k, candidate[k], source_map.get(k, False), interactive, vtype ) # Capabilities & Modes bool_keys = [f[0] for f in CAPABILITY_MAPPING] + ["is_text_model", "is_embedding_model"] + list(flags.keys()) for k in bool_keys: if k in candidate: final_entry[k], _ = review_field( k, candidate[k], source_map.get(k, False), interactive, "bool" ) # Stats final_entry["MMLU_Pro_score"], _ = review_field( "MMLU_Pro_score", candidate["MMLU_Pro_score"], False, interactive, "float" ) final_entry["seconds_per_output_token"], _ = review_field( "seconds_per_output_token", candidate["seconds_per_output_token"], False, interactive, "float" ) # Note if "note" in candidate: final_entry["note"], _ = review_field( "note", candidate["note"], False, interactive, "str" ) # Cleanup Nulls cleaned_entry = {k: v for k, v in final_entry.items() if v is not None} return cleaned_entry def update_model( model_id: str, existing_data: dict[str, Any], litellm_static: dict[str, Any], litellm_dynamic: dict[str, Any] | None = None, interactive: bool = True, ) -> dict[str, Any] | None: static_entry = None if model_id in litellm_static: static_entry = litellm_static[model_id] else: if "/" in model_id: model_name = model_id.split("/", 1)[1] if model_name in litellm_static: static_entry = litellm_static[model_name] dynamic_entry = litellm_dynamic.get(model_id) if litellm_dynamic else None if static_entry is None and dynamic_entry is None: print(f"\n WARNING: No LiteLLM data found for {model_id}") existing_entry = existing_data.get(model_id) new_entry = convert_and_review_model( model_id, static_entry, dynamic_entry, existing_entry, interactive=interactive, ) return new_entry def process_models( model_ids: list[str], existing_data: dict[str, Any], litellm_static: dict[str, Any], use_endpoint: bool = False, interactive: bool = True, skip_existing: bool = False, ) -> None: """ Process models and (if interactive is True) ask user whether to write each one to file. """ litellm_dynamic = None if use_endpoint: litellm_dynamic = fetch_dynamic_model_info(model_ids) # We work on the existing_data dictionary directly so we can save incrementally current_data_state = existing_data.copy() for model_id in model_ids: # Check if model exists and if we should skip it if skip_existing and model_id in current_data_state: print(f"\n [System] Model '{model_id}' already exists in file. Skipping.") continue new_entry = update_model( model_id, current_data_state, litellm_static, litellm_dynamic, interactive=interactive ) if new_entry: # Display Final Result print("\n" + "-"*30) print(f"FINAL JSON FOR: {model_id}") print(json.dumps(new_entry, indent=2)) print("-" * 30) # Ask user to write to file should_save = True if interactive: confirm = input(f"Write '{model_id}' to json file? [y/N]: ").strip().lower() should_save = confirm == 'y' if should_save: current_data_state[model_id] = new_entry save_data(current_data_state) else: print(f" [System] Skipped saving {model_id}.") # ============================================================================= # Main Entry Point # ============================================================================= def main(): parser = argparse.ArgumentParser( description="Update pz_models_information.json with external data sources", formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("model_ids", nargs="*", help="Model IDs to update") parser.add_argument("--use-endpoint", action="store_true", help="Fetch dynamic info") parser.add_argument("--non-interactive", action="store_true", help="Skip review and auto-save") parser.add_argument("--update-all", action="store_true", help="Update all existing") args = parser.parse_args() litellm_static = fetch_litellm_data() if not litellm_static: return existing_data = load_existing_data() skip_existing = False if args.update_all: model_ids = list(existing_data.keys()) elif args.model_ids: model_ids = args.model_ids skip_existing = True else: parser.print_help() return interactive = not args.non_interactive # Run the main processing loop process_models( model_ids, existing_data, litellm_static, use_endpoint=args.use_endpoint, interactive=interactive, skip_existing=skip_existing, ) print("\nAll operations complete.") if __name__ == "__main__": main() ================================================ FILE: src/palimpzest/__init__.py ================================================ import logging from palimpzest.constants import Cardinality, Model from palimpzest.core.data.context import Context, TextFileContext from palimpzest.core.data.dataset import Dataset from palimpzest.core.data.iter_dataset import ( AudioFileDataset, HTMLFileDataset, ImageFileDataset, IterDataset, MemoryDataset, PDFFileDataset, TextFileDataset, XLSFileDataset, ) from palimpzest.core.elements.groupbysig import GroupBySig from palimpzest.core.lib.schemas import AudioBase64, AudioFilepath, ImageBase64, ImageFilepath, ImageURL from palimpzest.policy import ( MaxQuality, MaxQualityAtFixedCost, MaxQualityAtFixedTime, MinCost, MinCostAtFixedQuality, MinTime, MinTimeAtFixedQuality, PlanCost, Policy, ) from palimpzest.query.processor.config import QueryProcessorConfig from palimpzest.validator.validator import Validator # Initialize the root logger logging.getLogger(__name__).addHandler(logging.NullHandler()) __all__ = [ # constants "Cardinality", "Model", # core "GroupBySig", "Context", "TextFileContext", "Dataset", "IterDataset", "AudioFileDataset", "MemoryDataset", "HTMLFileDataset", "ImageFileDataset", "PDFFileDataset", "TextFileDataset", "XLSFileDataset", # schemas "AudioBase64", "AudioFilepath", "ImageBase64", "ImageFilepath", "ImageURL", # policy "MaxQuality", "MaxQualityAtFixedCost", "MaxQualityAtFixedTime", "MinCost", "MinCostAtFixedQuality", "MinTime", "MinTimeAtFixedQuality", "PlanCost", "Policy", # query "QueryProcessorConfig", # validator "Validator", ] ================================================ FILE: src/palimpzest/agents/__init__.py ================================================ ================================================ FILE: src/palimpzest/agents/compute_agents.py ================================================ ================================================ FILE: src/palimpzest/agents/search_agents.py ================================================ import json import textwrap import time from collections.abc import Generator from typing import TYPE_CHECKING, Any from rich.console import Group from rich.live import Live from rich.markdown import Markdown from rich.rule import Rule from rich.text import Text if TYPE_CHECKING: import PIL.Image from smolagents.agent_types import handle_agent_output_types from smolagents.agents import ( ActionOutput, CodeAgent, FinalAnswerPromptTemplate, ManagedAgentPromptTemplate, PlanningPromptTemplate, PromptTemplates, RunResult, ToolOutput, populate_template, ) from smolagents.local_python_executor import fix_final_answer_code from smolagents.memory import ( ActionStep, FinalAnswerStep, PlanningStep, SystemPromptStep, TaskStep, Timing, TokenUsage, ToolCall, ) from smolagents.models import ( CODEAGENT_RESPONSE_FORMAT, ChatMessage, ChatMessageStreamDelta, MessageRole, agglomerate_stream_deltas, ) from smolagents.monitoring import YELLOW_HEX, LogLevel from smolagents.utils import ( AgentError, AgentExecutionError, AgentGenerationError, AgentMaxStepsError, AgentParsingError, extract_code_from_text, parse_code_blobs, truncate_content, ) from palimpzest.prompts import ( CODE_AGENT_SYSTEM_PROMPT, DATA_DISCOVERY_AGENT_INITIAL_PLAN_PROMPT, DATA_DISCOVERY_AGENT_REPORT_PROMPT, DATA_DISCOVERY_AGENT_TASK_PROMPT, DATA_DISCOVERY_AGENT_UPDATE_PLAN_POST_MESSAGES_PROMPT, DATA_DISCOVERY_AGENT_UPDATE_PLAN_PRE_MESSAGES_PROMPT, FINAL_ANSWER_POST_MESSAGES_PROMPT, FINAL_ANSWER_PRE_MESSAGES_PROMPT, ) # TODO: make this use memory the way you want class PZBaseAgent(CodeAgent): def __init__(self, run_id: str, context_description: str, *args, **kwargs): # memory_config = { # "vector_store": { # "provider": "chroma", # "config": { # "collection_name": f"palimpzest-memory-{self.__class__.__name__}", # "path": "./pz-chroma", # } # } # } # self.pz_memory = Memory.from_config(memory_config) self.run_id = run_id self.context_description = context_description super().__init__(*args, **kwargs) def write_memory_to_messages( self, summary_mode: bool = False, ) -> list[ChatMessage]: """ Reads past llm_outputs, actions, and observations or errors from the memory into a series of messages that can be used as input to the LLM. Adds a number of keywords (such as PLAN, error, etc) to help the LLM. """ messages = self.memory.system_prompt.to_messages(summary_mode=summary_mode) for memory_step in self.memory.steps: messages.extend(memory_step.to_messages(summary_mode=summary_mode)) return messages def _generate_planning_step( self, task, is_first_step: bool, step: int ) -> Generator[ChatMessageStreamDelta | PlanningStep]: start_time = time.time() if is_first_step: input_messages = [ ChatMessage( role=MessageRole.USER, content=[ { "type": "text", "text": populate_template( self.prompt_templates["planning"]["initial_plan"], variables={"task": task, "tools": self.tools, "managed_agents": self.managed_agents, "context_description": self.context_description}, ), } ], ) ] if self.stream_outputs and hasattr(self.model, "generate_stream"): plan_message_content = "" output_stream = self.model.generate_stream(input_messages, stop_sequences=[""]) # type: ignore input_tokens, output_tokens = 0, 0 with Live("", console=self.logger.console, vertical_overflow="visible") as live: for event in output_stream: if event.content is not None: plan_message_content += event.content live.update(Markdown(plan_message_content)) if event.token_usage: output_tokens += event.token_usage.output_tokens input_tokens = event.token_usage.input_tokens yield event else: plan_message = self.model.generate(input_messages, stop_sequences=[""]) plan_message_content = plan_message.content input_tokens, output_tokens = ( ( plan_message.token_usage.input_tokens, plan_message.token_usage.output_tokens, ) if plan_message.token_usage else (None, None) ) plan = textwrap.dedent( f"""Here are the facts I know and the plan of action that I will follow to solve the task:\n```\n{plan_message_content}\n```""" ) else: # Summary mode removes the system prompt and previous planning messages output by the model. # Removing previous planning messages avoids influencing too much the new plan. memory_messages = self.write_memory_to_messages(summary_mode=True) plan_update_pre = ChatMessage( role=MessageRole.SYSTEM, content=[ { "type": "text", "text": populate_template( self.prompt_templates["planning"]["update_plan_pre_messages"], variables={"task": task, "context_description": self.context_description} ), } ], ) plan_update_post = ChatMessage( role=MessageRole.USER, content=[ { "type": "text", "text": populate_template( self.prompt_templates["planning"]["update_plan_post_messages"], variables={ "task": task, "tools": self.tools, "managed_agents": self.managed_agents, "remaining_steps": (self.max_steps - step), "context_description": self.context_description, }, ), } ], ) input_messages = [plan_update_pre] + memory_messages + [plan_update_post] if self.stream_outputs and hasattr(self.model, "generate_stream"): plan_message_content = "" input_tokens, output_tokens = 0, 0 with Live("", console=self.logger.console, vertical_overflow="visible") as live: for event in self.model.generate_stream( input_messages, stop_sequences=[""], ): # type: ignore if event.content is not None: plan_message_content += event.content live.update(Markdown(plan_message_content)) if event.token_usage: output_tokens += event.token_usage.output_tokens input_tokens = event.token_usage.input_tokens yield event else: plan_message = self.model.generate(input_messages, stop_sequences=[""]) plan_message_content = plan_message.content if plan_message.token_usage is not None: input_tokens, output_tokens = ( plan_message.token_usage.input_tokens, plan_message.token_usage.output_tokens, ) plan = textwrap.dedent( f"""I still need to solve the task I was given:\n```\n{self.task}\n```\n\nHere are the facts I know and my new/updated plan of action to solve the task:\n```\n{plan_message_content}\n```""" ) log_headline = "Initial plan" if is_first_step else "Updated plan" self.logger.log(Rule(f"[bold]{log_headline}", style="orange"), Text(plan), level=LogLevel.INFO) yield PlanningStep( model_input_messages=input_messages, plan=plan, model_output_message=ChatMessage(role=MessageRole.ASSISTANT, content=plan_message_content), token_usage=TokenUsage(input_tokens=input_tokens, output_tokens=output_tokens), timing=Timing(start_time=start_time, end_time=time.time()), ) # def _curate_messages(self, input_messages: list[ChatMessage]) -> list[ChatMessage]: # """ # Try returning: # - System Prompt + task # - Current Plan # - Summary of previous conversation # """ # # initialize with the system prompt & original task # curated_messages = input_messages[:2] # # find the last planning step message # idx = len(self.memory.steps) - 1 # while idx > -1: # step = self.memory.steps[idx] # if isinstance(step, PlanningStep): # curated_messages.append(step.model_output_message) # break # idx -= 1 # # add summary of chat history # history = self.pz_memory.search("A condensed summary of the execution trace of the agent.", run_id=self.run_id) # for msg in history["results"]: # pass # return curated_messages def _step_stream(self, memory_step: ActionStep) -> Generator[ChatMessageStreamDelta | ActionOutput | ToolOutput]: """ Perform one step in the ReAct framework: the agent thinks, acts, and observes the result. Yields ChatMessageStreamDelta during the run if streaming is enabled. At the end, yields either None if the step is not final, or the final answer. """ memory_messages = self.write_memory_to_messages() input_messages = memory_messages.copy() ### Generate model output ### memory_step.model_input_messages = input_messages try: additional_args: dict[str, Any] = {} if self.grammar: additional_args["grammar"] = self.grammar if self._use_structured_outputs_internally: additional_args["response_format"] = CODEAGENT_RESPONSE_FORMAT if self.stream_outputs: output_stream = self.model.generate_stream( input_messages, stop_sequences=["", "Observation:", "Calling tools:"], **additional_args, ) chat_message_stream_deltas: list[ChatMessageStreamDelta] = [] with Live("", console=self.logger.console, vertical_overflow="visible") as live: for event in output_stream: chat_message_stream_deltas.append(event) live.update( Markdown(agglomerate_stream_deltas(chat_message_stream_deltas).render_as_markdown()) ) yield event chat_message = agglomerate_stream_deltas(chat_message_stream_deltas) memory_step.model_output_message = chat_message output_text = chat_message.content else: chat_message: ChatMessage = self.model.generate( input_messages, stop_sequences=["", "Observation:", "Calling tools:"], **additional_args, ) memory_step.model_output_message = chat_message output_text = chat_message.content self.logger.log_markdown( content=output_text, title="Output message of the LLM:", level=LogLevel.DEBUG, ) # This adds sequence to the history. # This will nudge ulterior LLM calls to finish with , thus efficiently stopping generation. if output_text and output_text.strip().endswith("```"): output_text += "" memory_step.model_output_message.content = output_text memory_step.token_usage = chat_message.token_usage memory_step.model_output = output_text except Exception as e: raise AgentGenerationError(f"Error in generating model output:\n{e}", self.logger) from e ### Parse output ### try: if self._use_structured_outputs_internally: code_action = json.loads(output_text)["code"] code_action = extract_code_from_text(code_action) or code_action else: code_action = parse_code_blobs(output_text) code_action = fix_final_answer_code(code_action) memory_step.code_action = code_action except Exception as e: error_msg = f"Error in code parsing:\n{e}\nMake sure to provide correct code blobs." raise AgentParsingError(error_msg, self.logger) from e memory_step.tool_calls = [ ToolCall( name="python_interpreter", arguments=code_action, id=f"call_{len(self.memory.steps)}", ) ] ### Execute action ### self.logger.log_code(title="Executing parsed code:", content=code_action, level=LogLevel.INFO) is_final_answer = False try: output, execution_logs, is_final_answer = self.python_executor(code_action) execution_outputs_console = [] if len(execution_logs) > 0: execution_outputs_console += [ Text("Execution logs:", style="bold"), Text(execution_logs), ] observation = "Execution logs:\n" + execution_logs except Exception as e: if hasattr(self.python_executor, "state") and "_print_outputs" in self.python_executor.state: execution_logs = str(self.python_executor.state["_print_outputs"]) if len(execution_logs) > 0: execution_outputs_console = [ Text("Execution logs:", style="bold"), Text(execution_logs), ] memory_step.observations = "Execution logs:\n" + execution_logs self.logger.log(Group(*execution_outputs_console), level=LogLevel.INFO) error_msg = str(e) if "Import of " in error_msg and " is not allowed" in error_msg: self.logger.log( "[bold red]Warning to user: Code execution failed due to an unauthorized import - Consider passing said import under `additional_authorized_imports` when initializing your CodeAgent.", level=LogLevel.INFO, ) raise AgentExecutionError(error_msg, self.logger) from e truncated_output = truncate_content(str(output)) observation += "Last output from code snippet:\n" + truncated_output memory_step.observations = observation # # TODO: add output to self.pz_memory # def get_role(msg_role): # return str(msg_role).split(".")[-1].lower() # messages = [ # {"role": get_role(memory_step.model_output_message.role), "content": memory_step.model_output_message.content}, # {"role": "user", "content": memory_step.observations}, # ] # self.pz_memory.add(messages, run_id=self.run_id, agent_id=self.name) execution_outputs_console += [ Text( f"{('Out - Final answer' if is_final_answer else 'Out')}: {truncated_output}", style=(f"bold {YELLOW_HEX}" if is_final_answer else ""), ), ] self.logger.log(Group(*execution_outputs_console), level=LogLevel.INFO) memory_step.action_output = output yield ActionOutput(output=output, is_final_answer=is_final_answer) def _run_stream( self, task: str, max_steps: int, images: list["PIL.Image.Image"] | None = None ) -> Generator[ActionStep | PlanningStep | FinalAnswerStep | ChatMessageStreamDelta]: """ Execute the agent. """ self.step_number = 1 returned_final_answer = False while not returned_final_answer and self.step_number <= self.max_steps: # Run a planning step if scheduled if self.planning_interval is not None and ( self.step_number == 1 or (self.step_number - 1) % self.planning_interval == 0 ): planning_start_time = time.time() planning_step = None for element in self._generate_planning_step( self.task, is_first_step=len(self.memory.steps) == 1, step=self.step_number ): # Don't use the attribute step_number here, because there can be steps from previous runs yield element planning_step = element assert isinstance(planning_step, PlanningStep) # Last yielded element should be a PlanningStep self.memory.steps.append(planning_step) planning_end_time = time.time() planning_step.timing = Timing( start_time=planning_start_time, end_time=planning_end_time, ) # Start action step! action_step_start_time = time.time() action_step = ActionStep( step_number=self.step_number, timing=Timing(start_time=action_step_start_time), observations_images=images, ) self.logger.log_rule(f"Step {self.step_number}", level=LogLevel.INFO) try: for output in self._step_stream(action_step): # Yield streaming deltas if not isinstance(output, (ActionOutput, ToolOutput)): # non-action, non-tool output yield output if isinstance(output, (ActionOutput, ToolOutput)) and output.is_final_answer: if self.final_answer_checks: self._validate_final_answer(output.output) returned_final_answer = True action_step.is_final_answer = True final_answer = output.output # handle final step except AgentGenerationError as e: # Agent generation errors are not caused by a Model error but an implementation error: so we should raise them and exit. raise e except AgentError as e: # Other AgentError types are caused by the Model, so we should log them and iterate. action_step.error = e finally: self._finalize_step(action_step) self.memory.steps.append(action_step) yield action_step self.step_number += 1 if not returned_final_answer and self.step_number == self.max_steps + 1: final_answer = self._handle_max_steps_reached(self.task, images) yield action_step yield FinalAnswerStep(handle_agent_output_types(final_answer)) def run( self, task: str, stream: bool = False, reset: bool = True, images: list["PIL.Image.Image"] | None = None, additional_args: dict | None = None, max_steps: int | None = None, ): """ Run the agent for the given task. Args: task (`str`): Task to perform. stream (`bool`): Whether to run in streaming mode. If `True`, returns a generator that yields each step as it is executed. You must iterate over this generator to process the individual steps (e.g., using a for loop or `next()`). If `False`, executes all steps internally and returns only the final answer after completion. reset (`bool`): Whether to reset the conversation or keep it going from previous run. images (`list[PIL.Image.Image]`, *optional*): Image(s) objects. additional_args (`dict`, *optional*): Any other variables that you want to pass to the agent run, for instance images or dataframes. Give them clear names! max_steps (`int`, *optional*): Maximum number of steps the agent can take to solve the task. if not provided, will use the agent's default value. Example: ```py from smolagents import CodeAgent agent = CodeAgent(tools=[]) agent.run("What is the result of 2 power 3.7384?") ``` """ max_steps = max_steps or self.max_steps self.task = task self.interrupt_switch = False if additional_args is not None: self.state.update(additional_args) self.task += f""" You have been provided with these additional arguments, that you can access using the keys as variables in your python code: {str(additional_args)}.""" self.memory.system_prompt = SystemPromptStep(system_prompt=self.system_prompt) if reset: self.memory.reset() self.monitor.reset() self.logger.log_task( content=self.task.strip(), subtitle=f"{type(self.model).__name__} - {(self.model.model_id if hasattr(self.model, 'model_id') else '')}", level=LogLevel.INFO, title=self.name if hasattr(self, "name") else None, ) self.memory.steps.append(TaskStep(task=self.task, task_images=images)) if getattr(self, "python_executor", None): self.python_executor.send_variables(variables=self.state) self.python_executor.send_tools({**self.tools, **self.managed_agents}) if stream: # The steps are returned as they are executed through a generator to iterate on. return self._run_stream(task=self.task, max_steps=max_steps, images=images) run_start_time = time.time() # Outputs are returned only at the end. We only look at the last step. steps = list(self._run_stream(task=self.task, max_steps=max_steps, images=images)) assert isinstance(steps[-1], FinalAnswerStep) output = steps[-1].output if self.return_full_result: total_input_tokens = 0 total_output_tokens = 0 correct_token_usage = True for step in self.memory.steps: if isinstance(step, (ActionStep, PlanningStep)): if step.token_usage is None: correct_token_usage = False break else: total_input_tokens += step.token_usage.input_tokens total_output_tokens += step.token_usage.output_tokens if correct_token_usage: token_usage = TokenUsage(input_tokens=total_input_tokens, output_tokens=total_output_tokens) else: token_usage = None if self.memory.steps and isinstance(getattr(self.memory.steps[-1], "error", None), AgentMaxStepsError): state = "max_steps_error" else: state = "success" messages = self.memory.get_full_steps() return RunResult( output=output, token_usage=token_usage, messages=messages, timing=Timing(start_time=run_start_time, end_time=time.time()), state=state, ) return output class PZBaseManagedAgent(PZBaseAgent): def __call__(self, task: str, **kwargs): """Adds additional prompting for the managed agent, runs it, and wraps the output. This method is called only by a managed agent. """ full_task = populate_template( self.prompt_templates["managed_agent"]["task"], variables=dict(name=self.name, task=task, context_description=self.context_description), ) result = self.run(full_task, **kwargs) report = result.output if isinstance(result, RunResult) else result answer = populate_template( self.prompt_templates["managed_agent"]["report"], variables=dict(name=self.name, final_answer=report) ) if self.provide_run_summary: answer += "\n\nFor more detail, find below a summary of this agent's work:\n\n" for message in self.write_memory_to_messages(summary_mode=True): content = message.content answer += "\n" + truncate_content(str(content)) + "\n---" answer += "\n" return answer class DataDiscoveryAgent(PZBaseManagedAgent): def __init__(self, run_id: str, context_description: str, *args, **kwargs): self.description = """A team member that will search a data repository to find files which help to answer your question. Ask him for all your questions that require searching a repository of relevant data. Provide him as much context as possible, in particular if you need to search on a specific timeframe! And don't hesitate to provide him with a complex search task, like finding a difference between two files. Your request must be a real sentence, not a keyword search! Like "Find me this information (...)" rather than a few keywords. """ prompt_templates = PromptTemplates( system_prompt=CODE_AGENT_SYSTEM_PROMPT, planning=PlanningPromptTemplate( initial_plan=DATA_DISCOVERY_AGENT_INITIAL_PLAN_PROMPT, update_plan_pre_messages=DATA_DISCOVERY_AGENT_UPDATE_PLAN_PRE_MESSAGES_PROMPT, update_plan_post_messages=DATA_DISCOVERY_AGENT_UPDATE_PLAN_POST_MESSAGES_PROMPT, ), managed_agent=ManagedAgentPromptTemplate(task=DATA_DISCOVERY_AGENT_TASK_PROMPT, report=DATA_DISCOVERY_AGENT_REPORT_PROMPT), final_answer=FinalAnswerPromptTemplate(pre_messages=FINAL_ANSWER_PRE_MESSAGES_PROMPT, post_messages=FINAL_ANSWER_POST_MESSAGES_PROMPT), ) super().__init__( *args, run_id=run_id, context_description=context_description, prompt_templates=prompt_templates, max_steps=20, verbosity_level=2, planning_interval=4, name="data_discovery_agent", description=self.description, provide_run_summary=True, **kwargs, ) self.prompt_templates["managed_agent"]["task"] += """Additionally, if after some searching you find out that you need more information to answer the question, you can use `final_answer` with your request for clarification as argument to request for more information.""" class SearchManagerAgent(PZBaseAgent): def __init__(self, run_id: str, context_description: str, *args, **kwargs): prompt_templates = PromptTemplates( system_prompt=CODE_AGENT_SYSTEM_PROMPT, planning=PlanningPromptTemplate( initial_plan=DATA_DISCOVERY_AGENT_INITIAL_PLAN_PROMPT, update_plan_pre_messages=DATA_DISCOVERY_AGENT_UPDATE_PLAN_PRE_MESSAGES_PROMPT, update_plan_post_messages=DATA_DISCOVERY_AGENT_UPDATE_PLAN_POST_MESSAGES_PROMPT, ), managed_agent=ManagedAgentPromptTemplate(task=DATA_DISCOVERY_AGENT_TASK_PROMPT, report=DATA_DISCOVERY_AGENT_REPORT_PROMPT), final_answer=FinalAnswerPromptTemplate(pre_messages=FINAL_ANSWER_PRE_MESSAGES_PROMPT, post_messages=FINAL_ANSWER_POST_MESSAGES_PROMPT), ) super().__init__( *args, run_id=run_id, context_description=context_description, prompt_templates=prompt_templates, max_steps=12, verbosity_level=2, additional_authorized_imports=["*"], planning_interval=4, return_full_result=True, **kwargs, ) # class ManagerAgent(CodeAgent): # def _step_stream(self, memory_step: ActionStep) -> Generator[ChatMessageStreamDelta | ActionOutput | ToolOutput]: # """ # Perform one step in the ReAct framework: the agent thinks, acts, and observes the result. # Yields ChatMessageStreamDelta during the run if streaming is enabled. # At the end, yields either None if the step is not final, or the final answer. # """ # raise NotImplementedError("This method should be implemented in child classes") ================================================ FILE: src/palimpzest/constants.py ================================================ ### This file contains constants used by Palimpzest ### from __future__ import annotations import os from enum import Enum import litellm from palimpzest.utils.model_info_helpers import ModelMetricsManager, predict_local_model_metrics class PromptStrategy(str, Enum): """ PromptStrategy describes the prompting technique to be used by a Generator when performing some task with a specified Model. """ # aggregation prompt strategies AGG = "aggregation" AGG_NO_REASONING = "aggregation-no-reasoning" # filter prompt strategies FILTER = "filter" FILTER_NO_REASONING = "filter-no-reasoning" FILTER_CRITIC = "filter-critic" FILTER_REFINE = "filter-refine" FILTER_MOA_PROPOSER = "filter-mixture-of-agents-proposer" FILTER_MOA_AGG = "filter-mixture-of-agents-aggregator" FILTER_SPLIT_PROPOSER = "filter-split-proposer" FILTER_SPLIT_MERGER = "filter-split-merger" # join prompt strategies JOIN = "join" JOIN_NO_REASONING = "join-no-reasoning" # map prompt strategies MAP = "map" MAP_NO_REASONING = "map-no-reasoning" MAP_CRITIC = "map-critic" MAP_REFINE = "map-refine" MAP_MOA_PROPOSER = "map-mixture-of-agents-proposer" MAP_MOA_AGG = "map-mixture-of-agents-aggregator" MAP_SPLIT_PROPOSER = "map-split-proposer" MAP_SPLIT_MERGER = "map-split-merger" def is_agg_prompt(self): return "aggregation" in self.value def is_filter_prompt(self): return "filter" in self.value def is_join_prompt(self): return "join" in self.value def is_map_prompt(self): return "map" in self.value def is_critic_prompt(self): return "critic" in self.value def is_refine_prompt(self): return "refine" in self.value def is_moa_proposer_prompt(self): return "mixture-of-agents-proposer" in self.value def is_moa_aggregator_prompt(self): return "mixture-of-agents-aggregator" in self.value def is_split_proposer_prompt(self): return "split-proposer" in self.value def is_split_merger_prompt(self): return "split-merger" in self.value def is_no_reasoning_prompt(self): return "no-reasoning" in self.value class Modality(str, Enum): TEXT = "text" IMAGE = "image" AUDIO = "audio" class AggFunc(str, Enum): COUNT = "count" AVERAGE = "average" SUM = "sum" MIN = "min" MAX = "max" class Cardinality(str, Enum): ONE_TO_ONE = "one-to-one" ONE_TO_MANY = "one-to-many" @classmethod def _missing_(cls, value): if value: normalized_value = "".join([x for x in value if x.isalpha()]).lower() for member in cls: normalized_member = "".join([x for x in member if x.isalpha()]).lower() if normalized_member == normalized_value: return member return cls.ONE_TO_ONE class PickOutputStrategy(str, Enum): CHAMPION = "champion" ENSEMBLE = "ensemble" AUDIO_EXTENSIONS = [".wav"] IMAGE_EXTENSIONS = [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff"] PDF_EXTENSIONS = [".pdf"] XLS_EXTENSIONS = [".xls", ".xlsx"] HTML_EXTENSIONS = [".html", ".htm"] # the number of seconds the parallel execution will sleep for while waiting for futures to complete PARALLEL_EXECUTION_SLEEP_INTERVAL_SECS = 0.3 # default PDF parser DEFAULT_PDF_PROCESSOR = "pypdf" # character limit for various IDs MAX_ID_CHARS = 10 # maximum number of rows to display in a table MAX_ROWS = 5 # maximum number of rows to parse from an HTML MAX_HTML_ROWS = 10000 def log_attempt_number(retry_state): """return the result of the last call attempt""" print(f"Retrying: {retry_state.attempt_number}...") # Palimpzest root directory PZ_DIR = os.path.join(os.path.expanduser("~"), ".palimpzest") # assume 500 MB/sec for local SSD scan time LOCAL_SCAN_TIME_PER_KB = 1 / (float(500) * 1024) # assume 30 GB/sec for sequential access of memory MEMORY_SCAN_TIME_PER_KB = 1 / (float(30) * 1024 * 1024) # assume 1 KB per record NAIVE_BYTES_PER_RECORD = 1024 # rough conversion from # of characters --> # of tokens; assumes 1 token ~= 4 chars TOKENS_PER_CHARACTER = 0.25 # rough estimate of the number of tokens the context is allowed to take up for LLAMA3 models LLAMA_CONTEXT_TOKENS_LIMIT = 6000 # a naive estimate for the input record size NAIVE_EST_SOURCE_RECORD_SIZE_IN_BYTES = 1_000_000 # a naive estimate for filter selectivity NAIVE_EST_FILTER_SELECTIVITY = 0.5 # a naive estimate for join selectivity NAIVE_EST_JOIN_SELECTIVITY = 0.5 # a naive estimate for the number of input tokens processed per record NAIVE_EST_NUM_INPUT_TOKENS = 1000 # a naive estimate for the number of output tokens processed per record NAIVE_EST_NUM_OUTPUT_TOKENS = 100 # a naive estimate for the number of groups returned by a group by NAIVE_EST_NUM_GROUPS = 3 # a naive estimate for the factor of increase (loosely termed "selectivity") for one-to-many cardinality operations NAIVE_EST_ONE_TO_MANY_SELECTIVITY = 2 # a naive estimate of the time it takes to extract the latex for an equation from an image file using Skema NAIVE_IMAGE_TO_EQUATION_LATEX_TIME_PER_RECORD = 10.0 # a naive estimate of the time it takes to extract the text from a PDF using a PDF processor NAIVE_PDF_PROCESSOR_TIME_PER_RECORD = 10.0 # whether or not to log LLM outputs LOG_LLM_OUTPUT = False # maximum number of models to use when user does not narrow optimization space MAX_AVAILABLE_MODELS = 5 class Model: """ Model describes the underlying LLM which should be used to perform some operation which requires invoking an LLM. """ # Registry of known models (maps value string to Model instance) _registry: dict[str, Model] = {} def __init__(self, model_id: str, api_base: str | None = None, **vllm_kwargs): self.metrics_manager = ModelMetricsManager() self.model_id = model_id self.api_base = api_base self.vllm_kwargs = vllm_kwargs if vllm_kwargs else {} # For vLLM models (api_base is set), try to get model info from litellm's local data if api_base is not None: self.model_specs = self._get_litellm_model_specs(model_id) else: self.model_specs = self.metrics_manager.get_model_metrics(model_id) if not self.model_specs: raise ValueError("Palimpzest currently does not contain information about this model.") Model._registry[model_id] = self def _get_litellm_model_specs(self, model_id: str) -> dict: """Get model specs from litellm's local model_cost data for vLLM models.""" # Use predict function to get quality, latency metrics, and capability flags predicted_metrics = predict_local_model_metrics(model_id) # Start with defaults, then overlay predicted values specs = { "is_text_model": True, "is_vision_model": False, "is_llama_model": False, "is_clip_model": False, "is_audio_model": False, "is_reasoning_model": False, "is_embedding_model": False, "is_text_image_multimodal_embedding_model": False, "is_vllm_model": True, # Mark as vLLM model "usd_per_input_token": 0.0, # Cost always 0 for local model "usd_per_output_token": 0.0, "seconds_per_output_token": predicted_metrics["seconds_per_output_token"], "MMLU_Pro_score": predicted_metrics["MMLU_Pro_score"], } # Overlay all flags detected from model name (including False values like is_text_model for embeddings) for key, value in predicted_metrics.items(): if key.startswith("is_"): specs[key] = value # Try litellm for additional capability detection (may not work for local models) try: if litellm.supports_vision(model=model_id): specs["is_vision_model"] = True except Exception: pass try: if litellm.supports_audio_input(model=model_id): specs["is_audio_model"] = True except Exception: pass return specs def __lt__(self, other): if isinstance(other, Model): return self.value < other.value if isinstance(other, str): return self.value < other return NotImplemented @classmethod def get_all_models(cls) -> list[Model]: return list(cls._registry.values()) @property def value(self) -> str: return self.model_id @property def provider(self) -> str | None: """Returns the provider string for this model.""" return self.model_specs.get("provider") @property def api_key_env_var(self) -> str | None: """ Returns the standard environment variable name for this provider's API key. """ if self.provider == "gemini": return "GEMINI_API_KEY" if os.getenv("GEMINI_API_KEY") else "GOOGLE_API_KEY" if self.provider == "azure": return "AZURE_API_KEY" if os.getenv("AZURE_API_KEY") else "AZURE_OPENAI_API_KEY" mapping = { "openai": "OPENAI_API_KEY", "vertex_ai": "GOOGLE_APPLICATION_CREDENTIALS", "anthropic": "ANTHROPIC_API_KEY", "together_ai": "TOGETHER_API_KEY", "hosted_vllm": "VLLM_API_KEY" } return mapping.get(self.provider) def __repr__(self) -> str: return self.value def __str__(self) -> str: return self.value def __eq__(self, other: object) -> bool: if isinstance(other, Model): return self.value == other.value if isinstance(other, str): return self.value == other return NotImplemented def __hash__(self) -> int: return hash(self.value) def is_llama_model(self) -> bool: return self.model_specs.get("is_llama_model", False) def is_vllm_model(self) -> bool: return self.model_specs.get("is_vllm_model", False) and self.api_base is not None def is_embedding_model(self) -> bool: return self.model_specs.get("is_embedding_model", False) def is_text_image_multimodal_embedding_model(self) -> bool: return self.model_specs.get("is_text_image_multimodal_embedding_model", False) def is_provider_vertex_ai(self) -> bool: return self.provider == "vertex_ai" def is_provider_anthropic(self) -> bool: return self.provider == "anthropic" def is_provider_google_ai_studio(self) -> bool: return self.provider == "gemini" def is_provider_openai(self) -> bool: return self.provider == "openai" def is_provider_azure(self) -> bool: return self.provider == "azure" def is_provider_together_ai(self) -> bool: return self.provider == "together_ai" def is_provider_deepseek(self) -> bool: return self.provider == "deepseek" def is_provider_ollama(self) -> bool: return self.provider == "ollama" def is_model_gemini(self) -> bool: return "gemini" in self.value.lower() def get_model_name(self) -> str: return self.value.split("/")[-1] if "/" in self.value else self.value def is_o_model(self) -> bool: return self.model_specs.get("is_o_model", False) def is_gpt_5_model(self) -> bool: return self.model_specs.get("is_gpt_5_model", False) def is_reasoning_model(self) -> bool: return self.model_specs.get("is_reasoning_model", False) def is_text_model(self) -> bool: return self.model_specs.get("is_text_model", False) def is_vision_model(self) -> bool: return self.model_specs.get("is_vision_model", False) def is_audio_model(self) -> bool: return self.model_specs.get("is_audio_model", False) def is_text_image_multimodal_model(self) -> bool: return self.is_text_model() and self.is_vision_model() def is_text_audio_multimodal_model(self) -> bool: return self.is_audio_model() and self.is_text_model() def supports_prompt_caching(self) -> bool: return (self.is_provider_anthropic() or self.is_provider_google_ai_studio() or self.is_provider_vertex_ai or self.is_provider_openai() or self.is_provider_azure()) \ and self.model_specs.get("supports_prompt_caching", False) def get_usd_per_input_token(self) -> float: return self.model_specs.get("usd_per_input_token", 0.0) def get_usd_per_audio_input_token(self) -> float: return self.model_specs.get("usd_per_audio_input_token", self.get_usd_per_input_token()) # forward-looking, TODO: default value discussion def get_usd_per_image_input_token(self) -> float: return self.model_specs.get("usd_per_image_input_token", self.get_usd_per_input_token()) def get_usd_per_cache_read_token(self) -> float: return self.model_specs.get("usd_per_cache_read_token", self.get_usd_per_input_token()) def get_usd_per_audio_cache_read_token(self) -> float: return self.model_specs.get("usd_per_audio_cache_read_token", self.get_usd_per_cache_read_token()) def get_usd_per_image_cache_read_token(self) -> float: return self.model_specs.get("usd_per_image_cache_read_token", self.get_usd_per_cache_read_token()) # forward looking; Gemini explicit def get_usd_per_cached_token_per_hour(self) -> float: return self.model_specs.get("usd_per_cached_token_per_hour", 0.0) def get_usd_per_cache_creation_token(self) -> float: return self.model_specs.get("usd_per_cache_creation_token", 0.0) def get_usd_per_output_token(self) -> float: return self.model_specs.get("usd_per_output_token", 0.0) # forward-looking def get_usd_per_audio_cache_creation_token(self) -> float: return self.model_specs.get("usd_per_audio_cache_creation_token", 0.0) # forward-looking def get_usd_per_image_cache_creation_token(self) -> float: return self.model_specs.get("usd_per_image_cache_creation_token", 0.0) def get_seconds_per_output_token(self) -> float: return self.model_specs.get("seconds_per_output_token", 0.0) def get_overall_score(self) -> float: return self.model_specs.get("MMLU_Pro_score", 0.0) # TODO: investigate which (if any llama3 models are still supported by TogetherAI) # Model.LLAMA3_2_3B = Model("together_ai/meta-llama/Llama-3.2-3B-Instruct-Turbo") - seems to be deprecated Model.LLAMA3_1_8B = Model("together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo") Model.LLAMA3_3_70B = Model("together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo") Model.LLAMA3_2_90B_V = Model("together_ai/meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo") Model.DEEPSEEK_V3 = Model("together_ai/deepseek-ai/DeepSeek-V3") Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B = Model("together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B") Model.GPT_4o = Model("openai/gpt-4o-2024-08-06") Model.GPT_4o_MINI = Model("openai/gpt-4o-mini-2024-07-18") Model.GPT_4_1 = Model("openai/gpt-4.1-2025-04-14") Model.GPT_4_1_MINI = Model("openai/gpt-4.1-mini-2025-04-14") Model.GPT_4_1_NANO = Model("openai/gpt-4.1-nano-2025-04-14") Model.GPT_5 = Model("openai/gpt-5-2025-08-07") Model.GPT_5_MINI = Model("openai/gpt-5-mini-2025-08-07") Model.GPT_5_NANO = Model("openai/gpt-5-nano-2025-08-07") Model.GPT_5_2 = Model("openai/gpt-5.2-2025-12-11") Model.o4_MINI = Model("openai/o4-mini-2025-04-16") # noqa: N815 # Model.CLAUDE_3_5_SONNET = Model("anthropic/claude-3-5-sonnet-20241022") - retired 10/28/2025 Model.CLAUDE_3_7_SONNET = Model("anthropic/claude-3-7-sonnet-20250219") Model.CLAUDE_4_SONNET = Model("anthropic/claude-sonnet-4-20250514") Model.CLAUDE_4_5_SONNET = Model("anthropic/claude-sonnet-4-5-20250929") Model.CLAUDE_3_5_HAIKU = Model("anthropic/claude-3-5-haiku-20241022") Model.CLAUDE_4_5_HAIKU = Model("anthropic/claude-haiku-4-5-20251001") Model.GEMINI_3_0_PRO = Model("vertex_ai/gemini-3-pro-preview") # image Model.GEMINI_3_0_FLASH = Model("vertex_ai/gemini-3-flash-preview") # Text, Image, Video, Audio, and PDF Model.GEMINI_2_0_FLASH = Model("vertex_ai/gemini-2.0-flash") Model.GEMINI_2_5_FLASH = Model("vertex_ai/gemini-2.5-flash") Model.GEMINI_2_5_PRO = Model("vertex_ai/gemini-2.5-pro") Model.GOOGLE_GEMINI_3_0_PRO = Model("gemini/gemini-3-pro-preview") Model.GOOGLE_GEMINI_3_0_FLASH = Model("gemini/gemini-3-flash-preview") Model.GOOGLE_GEMINI_2_5_FLASH = Model("gemini/gemini-2.5-flash") Model.GOOGLE_GEMINI_2_5_FLASH_LITE = Model("gemini/gemini-2.5-flash-lite") Model.GOOGLE_GEMINI_2_5_PRO = Model("gemini/gemini-2.5-pro") Model.LLAMA_4_MAVERICK = Model("vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas") Model.GPT_4o_AUDIO_PREVIEW = Model("openai/gpt-4o-audio-preview") Model.GPT_4o_MINI_AUDIO_PREVIEW = Model("openai/gpt-4o-mini-audio-preview") Model.AZURE_GPT_4o = Model("azure/gpt-4o-2024-08-06") Model.AZURE_GPT_4o_MINI = Model("azure/gpt-4o-mini-2024-07-18") Model.AZURE_GPT_4_1 = Model("azure/gpt-4.1-2025-04-14") Model.AZURE_GPT_4_1_MINI = Model("azure/gpt-4.1-mini-2025-04-14") Model.AZURE_GPT_4_1_NANO = Model("azure/gpt-4.1-nano-2025-04-14") Model.AZURE_o4_MINI = Model("azure/o4-mini-2025-04-16") # noqa: N815 Model.AZURE_GPT_4o_AUDIO_PREVIEW = Model("azure/gpt-4o-audio-preview") Model.AZURE_GPT_4o_MINI_AUDIO_PREVIEW = Model("azure/gpt-4o-mini-audio-preview") Model.TEXT_EMBEDDING_3_SMALL = Model("openai/text-embedding-3-small") Model.CLIP_VIT_B_32 = Model("clip-ViT-B-32") Model.NOMIC_EMBED_TEXT = Model("ollama/nomic-embed-text") #### MODEL PERFORMANCE & COST METRICS #### # Overall model quality is computed using MMLU-Pro; multi-modal models currently use the same score for vision # - in the future we should split quality for vision vs. multi-modal vs. text # - code quality was computed using HumanEval, but that benchmark is too easy and should be replaced. # - https://huggingface.co/spaces/TIGER-Lab/MMLU-Pro # - https://www.vals.ai/benchmarks/mmlu_pro-08-12-2025 # # Cost is presented in terms of USD / token for input tokens and USD / token for # generated tokens. # # Time is presented in seconds per output token. I grabbed some semi-recent estimates # from the internet for this quick POC, but we can and should do more to model these # values more precisely: # - https://artificialanalysis.ai/models/llama-3-1-instruct-8b # # Model metrics now fetched from singular json file curated_model_info.json ================================================ FILE: src/palimpzest/core/__init__.py ================================================ ================================================ FILE: src/palimpzest/core/data/__init__.py ================================================ ================================================ FILE: src/palimpzest/core/data/context.py ================================================ from __future__ import annotations import os import re from abc import ABC from typing import Callable from pydantic import BaseModel from smolagents import CodeAgent, LiteLLMModel from palimpzest.core.data import context_manager from palimpzest.core.data.dataset import Dataset from palimpzest.core.lib.schemas import create_schema_from_fields, union_schemas from palimpzest.query.operators.logical import ComputeOperator, ContextScan, LogicalOperator, SearchOperator from palimpzest.utils.hash_helpers import hash_for_id PZ_INSTRUCTION = """\n\nYou are a CodeAgent who is a specialist at writing declarative AI programs with the Palimpzest (PZ) library. Palimpzest is a programming framework which provides you with **semantic operators** (e.g. semantic maps, semantic filters, etc.) which are like their traditional counterparts, except they can execute instructions provided in natural language. For example, if you wanted to write a program to extract the title and abstract from a directory of papers, you could write the following in PZ: ``` import palimpzest as pz from dotenv import load_dotenv # Load environment variables from .env file load_dotenv() # Define columns for semantic map (sem_map) operation; each column is specified # with a dictionary containing the following keys: # - "name": the name of the field to compute # - "type": the type of the field to compute # - "description": the natural language description of the field paper_cols = [ {"name": "title", "type": str, "description": "the title of the paper"}, {"name": "abstract", "type": str, "description": "the paper's abstract"}, ] # construct the data processing pipeline with PZ ds = pz.TextFileDataset(id="papers", path="path/to/papers") ds = ds.sem_map(cols) # optimize and execute the PZ program validator = pz.Validator() config = pz.QueryProcessorConfig( policy=pz.MaxQuality(), execution_strategy="parallel", max_workers=20, progress=True, ) output = ds.optimize_and_run(config=config, validator=validator) # write the execution stats to json output.execution_stats.to_json("pz_program_stats.json") # write the output to a CSV and print the output CSV filepath so the user knows where to find it output_filepath = "pz_program_output.csv" output.to_df().to_csv(output_filepath, index=False) print(f"Results at: {output_filepath}") ``` To initialize a dataset in PZ, simply provide the path to a directory to `pz.TextFileDirectory()` (if your data contains text-based files). For example: ``` import palimpzest as pz from dotenv import load_dotenv # Load environment variables from .env file load_dotenv() ds = pz.TextFileDataset(id="files", path="path/to/files") ``` Palimpzest has two primary **semantic operators** which you can use to construct data processing pipelines: - sem_filter(predicate: str): executes a semantic filter specified by the natural language predicate on a given PZ dataset - sem_map(cols: list[dict]): executes a semantic map to compute the `cols` on a given PZ dataset As a second example, consider the following PZ program which filters for papers about batteries that are from MIT and computes a summary for each one: ``` import palimpzest as pz from dotenv import load_dotenv # Load environment variables from .env file load_dotenv() # construct the PZ program ds = pz.TextFileDataset(id="papers", path="path/to/research-papers") ds = ds.sem_filter("The paper is about batteries") ds = ds.sem_filter("The paper is from MIT") ds = ds.sem_map([{"name": "summary", "type": str, "description": "A summary of the paper"}]) # optimize and execute the PZ program validator = pz.Validator() config = pz.QueryProcessorConfig( policy=pz.MaxQuality(), execution_strategy="parallel", max_workers=20, progress=True, ) output = ds.optimize_and_run(config=config, validator=validator) # write the execution stats to json output.execution_stats.to_json("pz_program_stats.json") # write the output to a CSV and print the output CSV filepath so the user knows where to find it output_filepath = "pz_program_output.csv" output.to_df().to_csv(output_filepath, index=False) print(f"Results at: {output_filepath}") ``` Be sure to always: - execute your program using the `.optimize_and_run()` format shown above - call `output.execution_stats.to_json("pz_program_stats.json")` to write execution statistics to disk - write your output to CSV and print where you wrote it! """ class Context(Dataset, ABC): """ The `Context` class is an abstract base class for root `Datasets` whose data is accessed via user-defined methods. Classes which inherit from this class must implement two methods: - `list_filepaths()`: which lists the files that the `Context` has access to. - `read_filepath(path: str)`: which reads the file corresponding to the given `path`. A `Context` is a special type of `Dataset` that represents a view over an underlying `Dataset`. Each `Context` has a `name` which uniquely identifies it, as well as a natural language `description` of the data / computation that the `Context` represents. Similar to `Dataset`s, `Context`s can be lazily transformed using functions such as `sem_filter`, `sem_map`, `sem_join`, etc., and they may be materialized or unmaterialized. """ def __init__( self, id: str, description: str, operator: LogicalOperator, schema: type[BaseModel] | None = None, sources: list[Context] | Context | None = None, materialized: bool = False, ) -> None: """ Constructor for the `Context` class. Args: id (`str`): a string identifier for the `Context` description (`str`): the description of the data contained within the `Context` operator (`LogicalOperator`): The `LogicalOperator` used to compute this `Context`. schema: (`type[BaseModel] | None`): The schema of this `Context`. sources (`list[Context] | Context | None`): The (list of) `Context(s)` which are input(s) to the operator used to compute this `Context`. materialized (`bool`): True if the `Context` has been computed, False otherwise """ # set the description self._description = description # set the materialization status self._materialized = materialized # compute schema and call parent constructor if schema is None: schema = create_schema_from_fields([{"name": "context", "description": "The context", "type": str}]) super().__init__(sources=sources, operator=operator, schema=schema, id=id) # set the tools associated with this Context self._tools = [getattr(self, attr) for attr in dir(self) if attr.startswith("tool_")] # add Context to ContextManager cm = context_manager.ContextManager() cm.add_context(self) @property def description(self) -> str: """The string containing all of the information computed for this `Context`""" return self._description @property def materialized(self) -> bool: """The boolean which specifies whether the `Context` has been computed or not""" return self._materialized @property def tools(self) -> list[Callable]: """The list of tools associated with this `Context`""" return self._tools def __str__(self) -> str: return f"Context(id={self.id}, description={self.description:20s}, materialized={self.materialized})" def set_description(self, description: str) -> None: """ Update the context's description. """ self._description = description def set_materialized(self, materialized: str) -> None: """ Update the context's materialization status. """ self._materialized = materialized def compute(self, instruction: str) -> Context: # construct new description and output schema new_id = hash_for_id(instruction) new_description = f"Parent Context ID: {self.id}\n\nThis Context is the result of computing the following instruction on the parent context.\n\nINSTRUCTION: {instruction}\n\n" inter_schema = create_schema_from_fields([{"name": f"result-{new_id}", "desc": "The result from computing the instruction on the input Context", "type": str}]) new_output_schema = union_schemas([self.schema, inter_schema]) # construct logical operator operator = ComputeOperator( input_schema=self.schema, output_schema=new_output_schema, context_id=new_id, instruction=instruction, ) return Context(id=new_id, description=new_description, operator=operator, sources=[self], materialized=False) def search(self, search_query: str) -> Context: # construct new description and output schema new_id = hash_for_id(search_query) new_description = f"Parent Context ID: {self.id}\n\nThis Context is the result of searching the parent context for information related to the following query.\n\nSEARCH QUERY: {search_query}\n\n" # construct logical operator operator = SearchOperator( input_schema=self.schema, output_schema=self.schema, context_id=new_id, search_query=search_query, ) return Context(id=new_id, description=new_description, operator=operator, sources=[self], materialized=False) class TextFileContext(Context): def __init__(self, path: str, id: str, description: str) -> None: """ Constructor for the `TextFileContext` class. Args: path (str): The path to the file id (str): a string identifier for the `Context` description (str): The description of the data contained within the `Context` kwargs (dict): Keyword arguments containing the `Context's` id and description. """ # check that path is a valid file or directory assert os.path.isfile(path) or os.path.isdir(path), f"Path {path} is not a file nor a directory" # get list of filepaths self.filepaths = [] if os.path.isfile(path): self.filepaths = [path] else: self.filepaths = [] for root, _, files in os.walk(path): for file in files: fp = os.path.join(root, file) self.filepaths.append(fp) self.filepaths = sorted(self.filepaths) # call parent constructor to set id, operator, and schema schema = create_schema_from_fields([{"name": "context", "desc": "The context", "type": str}]) super().__init__( id=id, description=description, operator=ContextScan(context=self, output_schema=schema), schema=schema, materialized=True, ) def _check_filter_answer_text(self, answer_text: str) -> dict | None: """ Return {"passed_operator": True} if and only if "true" is in the answer text. Return {"passed_operator": False} if and only if "false" is in the answer text. Otherwise, return None. """ # NOTE: we may be able to eliminate this condition by specifying this JSON output in the prompt; # however, that would also need to coincide with a change to allow the parse_answer_fn to set "passed_operator" if "true" in answer_text.lower(): return {"passed_operator": True} elif "false" in answer_text.lower(): return {"passed_operator": False} elif "yes" in answer_text.lower(): return {"passed_operator": True} return None def _parse_filter_answer(self, completion_text: str) -> dict[str, list]: """Extract the answer from the completion object for filter operations.""" # if the model followed the default instructions, the completion text will place # its answer between "ANSWER:" and "---" regex = re.compile("answer:(.*?)---", re.IGNORECASE | re.DOTALL) matches = regex.findall(completion_text) if len(matches) > 0: answer_text = matches[0].strip() field_answers = self._check_filter_answer_text(answer_text) if field_answers is not None: return field_answers # if the first regex didn't find an answer, try taking all the text after "ANSWER:" regex = re.compile("answer:(.*)", re.IGNORECASE | re.DOTALL) matches = regex.findall(completion_text) if len(matches) > 0: answer_text = matches[0].strip() field_answers = self._check_filter_answer_text(answer_text) if field_answers is not None: return field_answers # finally, try taking all of the text; throw an exception if this doesn't work field_answers = self._check_filter_answer_text(completion_text) if field_answers is None: raise Exception(f"Could not parse answer from completion text: {completion_text}") return field_answers # def tool_list_filepaths(self) -> list[str]: # """ # This tool returns the list of all of the filepaths which the `Context` has access to. # Args: # None # Returns: # list[str]: A list of file paths for all files in the `Context`. # """ # return self.filepaths # def tool_read_filepath(self, path: str) -> str: # """ # This tool takes a filepath (`path`) as input and returns the content of the file as a string. # It handles both CSV files and html / regular text files. It does not handle images. # Args: # path (str): The path to the file to read. # Returns: # str: The content of the file as a string. # """ # if path.endswith(".csv"): # return pd.read_csv(path, encoding="ISO-8859-1").to_string(index=False) # with open(path, encoding='utf-8') as file: # content = file.read() # return content def tool_execute_semantic_operators(self, instruction: str) -> str: """ This tool takes an `instruction` as input and invokes an expert to write a semantic data processing pipeline to execute the instruction. The tool returns the path to a CSV file which contains the output of the pipeline. For example, the tool could be invoked as follows to extract the title and abstract from a dataset of research papers: ``` instruction = "Write a program to extract the title and abstract from each research paper" result_csv_filepath = tool_execute_semantic_operators(instruction) ``` Args: instruction: The instruction specifying the semantic data processing pipeline that you need to execute. Returns: str: the filepath to the CSV containing the output from running the data processing pipeline. """ from smolagents import tool @tool def tool_list_filepaths() -> list[str]: """ This tool returns the list of all of the filepaths which the `Context` has access to. NOTE: You may want to execute this before writing your PZ program to determine where the data lives. Args: None Returns: list[str]: A list of file paths for all files in the `Context`. """ return self.filepaths agent = CodeAgent( model=LiteLLMModel(model_id="openai/o1", api_key=os.getenv("ANTHROPIC_API_KEY")), tools=[tool_list_filepaths], max_steps=20, planning_interval=4, add_base_tools=False, return_full_result=True, additional_authorized_imports=["dotenv", "json", "palimpzest", "pandas"], instructions=PZ_INSTRUCTION, ) result = agent.run(instruction) response = result.output return response ================================================ FILE: src/palimpzest/core/data/context_manager.py ================================================ from __future__ import annotations import os import pickle import chromadb import chromadb.utils.embedding_functions as embedding_functions import tiktoken from palimpzest.constants import PZ_DIR from palimpzest.core.data import context class ContextNotFoundError(Exception): pass class ContextManager: """ This class manages the long-term storage of `Contexts`. Each new `Context` is added to the `ContextManager` and serialized to disk. `Contexts` are also indexed, which enables PZ to search for `Context(s)` which may support `search()` and `compute()` operations. """ def __init__(self): # create directory with serialized contexts (if it doesn't already exist) self.context_dir = os.path.join(PZ_DIR, "contexts") os.makedirs(self.context_dir, exist_ok=True) # create vector store (if it doesn't already exist) self.chroma_dir = os.path.join(PZ_DIR, "chroma") os.makedirs(self.chroma_dir, exist_ok=True) self.chroma_client = chromadb.PersistentClient(self.chroma_dir) # pick embedding function based on presence of API key(s) self.emb_fn = None if os.getenv("OPENAI_API_KEY"): self.emb_fn = embedding_functions.OpenAIEmbeddingFunction( api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small" ) self.index = self.chroma_client.get_or_create_collection("contexts", embedding_function=self.emb_fn) @staticmethod def from_pkl(path: str) -> context.Context: """Load a `Context` from its serialized pickle file.""" with open(path, "rb") as f: context = pickle.load(f) return context @staticmethod def to_pkl(context: context.Context, path: str) -> None: """Write the given `Context` to a pickle file at the provided `path`.""" with open(path, "wb") as f: pickle.dump(context, f) def num_tokens_from_string(self, string: str, encoding_name: str) -> int: """Returns the number of tokens in a text string.""" encoding = tiktoken.get_encoding(encoding_name) num_tokens = len(encoding.encode(string)) return num_tokens def add_context(self, context: context.Context, update: bool = False) -> None: """ Add the new `Context` to the `ContextManager` by serializing and writing it to disk. Args: context (`Context`): the context to add to the `ContextManager` update (`bool`): whether or not to update an existing context TODO: track cost """ # return early if the context already exists and we're not performing an update id = context.id context_path = os.path.join(self.context_dir, f"{id}.pkl") if os.path.exists(context_path) and update is False: return # write the context to disk ContextManager.to_pkl(context, context_path) # compute number of tokens in context.description description = context.description while self.num_tokens_from_string(description, "cl100k_base") > 8192: description = description[:int(0.9*len(description))] # add context to vector store context_embeddings = self.emb_fn([description]) context_payload = { "ids": [context.id], "embeddings": context_embeddings, "metadatas": [{"id": context.id, "materialized": context.materialized}], "documents": [context.description], } if update: self.index.update(**context_payload) else: self.index.add(**context_payload) def update_context(self, id: str, description: str, materialized: bool = True) -> None: """ Update an existing `Context` with the given `id` to have the given `description`. Args: id (str): the id of the updated `Context` description (str): the update to the description for the specified `Context` materialized (bool): boolean to set the materialization status of the `Context` Raises: ContextNotFoundError: if the given `id` doesn't point to a `Context` in the `ContextManger`. """ context = self.get_context(id) new_description = context.description + description # TODO: should description have RESULT replaced on update? as opposed to appending? should description be some pydantic BaseModel? context.set_description(new_description) context.set_materialized(materialized) self.add_context(context, update=True) def get_context(self, id: str) -> context.Context: """ Returns the `Context` specified by the given `id`. Args: id (str): the id of the retrieved `Context` Returns: `Context`: the specified `Context`. """ context_path = os.path.join(self.context_dir, f"{id}.pkl") try: return ContextManager.from_pkl(context_path) except FileNotFoundError as err: raise ContextNotFoundError from err def search_context(self, query: str, k: int = 1, where: dict | None = None) -> list[context.Context]: """ Returns the top-k most relevant `Context(s)` for the given query. If provided, the where dictionary will be used to filter the search results. TODO: 3) update CostModel to account for benefit of using existing Context(s) --- 4) unit test 5) track cost """ # embed the search query query_embeddings = self.emb_fn([query]) # look up ids of most similar contexts results = self.index.query( query_embeddings=query_embeddings, n_results=k, where=where, ) ids = results["ids"][0] # load and return Context objects contexts = [] for id in ids: context_path = os.path.join(self.context_dir, f"{id}.pkl") contexts.append(ContextManager.from_pkl(context_path)) return contexts ================================================ FILE: src/palimpzest/core/data/dataset.py ================================================ from __future__ import annotations import warnings from collections.abc import Iterator from typing import Callable from chromadb.api.models.Collection import Collection from pydantic import BaseModel from palimpzest.constants import AggFunc, Cardinality from palimpzest.core.elements.filters import Filter from palimpzest.core.elements.groupbysig import GroupBySig from palimpzest.core.lib.schemas import create_schema_from_fields, project, relax_schema, union_schemas from palimpzest.policy import construct_policy_from_kwargs from palimpzest.query.operators.logical import ( Aggregate, ConvertScan, Distinct, FilteredScan, GroupByAggregate, JoinOp, LimitScan, LogicalOperator, Project, TopKScan, ) from palimpzest.query.processor.config import QueryProcessorConfig from palimpzest.utils.hash_helpers import hash_for_serialized_dict from palimpzest.validator.validator import Validator # TODO?: remove `schema` from `Dataset` and access it from `operator`? # - Q: how do you handle datasets with multiple sources? # - for joins the operator should have the union'ed schema # - but for Contexts it may be trickier class Dataset: """ A `Dataset` represents a collection of structured or unstructured data that can be processed and transformed. Each `Dataset` is either a "root" `Dataset` (which yields data items) or it is the result of performing data processing operation(s) on root `Dataset(s)`. Users can perform computations on a `Dataset` in a lazy or eager fashion. Applying functions such as `sem_filter`, `sem_map`, `sem_join`, `sem_agg`, etc. will lazily create a new `Dataset`. Users can invoke the `run()` method to execute the computation and retrieve a materialized `Dataset`. Materialized `Dataset`s can be processed further, or their results can be retrieved using `.get()`. A root `Dataset` must subclass at least one of `pz.IterDataset`, `pz.IndexDataset`, or `pz.Context`. Each of these classes supports a different access pattern: - `pz.IterDataset`: supports accessing data via iteration - Ex: iterating over a list of PDFs - Ex: iterating over rows in a DataFrame - `pz.IndexDataset`: supports accessing data via point lookups / queries - Ex: querying a vector database - Ex: querying a SQL database - `pz.Context`: supports accessing data with an agent - Ex: processing a set of CSV files with a data science agent - Ex: processing time series data with a data cleaning agent A root `Dataset` may subclass more than one of the aforementioned classes. For example, the root `Dataset` for a list of files may inherit from `pz.IterDataset` and `pz.IndexDataset` to support iterating over the files and performing point lookups for individual files. For details on how to create your own root `Dataset`, please see: TODO """ def __init__( self, sources: list[Dataset] | Dataset | None, operator: LogicalOperator, schema: type[BaseModel] | None = None, id: str | None = None, ) -> None: """ Initialize a `Dataset` with one or more `sources` and the operator that is being applied. Root `Datasets` subclass `pz.IterDataset`, `pz.IndexDataset`, and/or `pz.Context` and use their own constructors. Args: sources (`list[Dataset] | Dataset`): The (list of) `Dataset(s)` which are input(s) to the operator used to compute this `Dataset`. operator (`LogicalOperator`): The `LogicalOperator` used to compute this `Dataset`. schema (type[`BaseModel`] | None): The schema of this `Dataset`. id (str | None): an identifier for this `Dataset` provided by the user Raises: ValueError: if `sources` is not a `Dataset` or list of `Datasets` """ # set sources self._sources = [] if isinstance(sources, list): self._sources = sources elif isinstance(sources, Dataset): self._sources = [sources] elif sources is not None: raise ValueError("Dataset sources must be another Dataset or a list of Datasets. For root Datasets, you must subclass pz.IterDataset, pz.IndexDataset, or pz.Context.") # set the logical operator and schema self._operator: LogicalOperator = operator self._schema = schema # compute the dataset id self._id = self._compute_dataset_id() if id is None else id @property def id(self) -> str: """The string identifier for this `Dataset`""" return self._id @property def schema(self) -> type[BaseModel]: """The Pydantic model defining the schema of this `Dataset`""" return self._schema @property def is_root(self) -> bool: return len(self._sources) == 0 def __str__(self) -> str: return f"Dataset(schema={self._schema}, id={self._id}, op_id={self._operator.get_logical_op_id()})" def __iter__(self) -> Iterator[Dataset]: for source in self._sources: yield from source yield self def _compute_dataset_id(self) -> str: """ Compute the identifier for this `Dataset`. The ID is uniquely defined by the operation(s) applied to the `Dataset's` sources. """ return hash_for_serialized_dict({ "source_ids": [source.id for source in self._sources], "logical_op_id": self._operator.get_logical_op_id(), }) def _set_root_datasets(self, new_root_datasets: dict[str, Dataset]) -> None: """ Update the root dataset(s) for this dataset with the `new_root_datasets`. This is used during optimization to reuse the same physical plan while running it on a train dataset. Args: new_root_datasets (dict[str, Dataset]): the new root datasets for this dataset. """ new_sources = [] for old_source in self._sources: if old_source.id in new_root_datasets: new_sources.append(new_root_datasets[old_source.id]) else: old_source._set_root_datasets(new_root_datasets) new_sources.append(old_source) self._sources = new_sources # TODO: the entire way (unique) logical op ids are computed and stored needs to be rethought def _generate_unique_logical_op_ids(self, topo_idx: int | None = None) -> None: """ Generate unique operation IDs for all operators in this dataset and its sources. This is used to ensure that each operator can be uniquely identified during execution. """ # generate the unique op ids for all sources' operators for source in self._sources: topo_idx = source._generate_unique_logical_op_ids(topo_idx) topo_idx += 1 # if topo_idx is None, this is the first call, so we initialize it to 0 if topo_idx is None: topo_idx = 0 # compute this operator's unique operator id this_unique_logical_op_id = f"{topo_idx}-{self._operator.get_logical_op_id()}" # update the unique logical op id for this operator self._operator.set_unique_logical_op_id(this_unique_logical_op_id) # return the current unique full_op_id for this operator return topo_idx # TODO def _resolve_depends_on(self, depends_on: list[str]) -> list[str]: """ TODO: resolve the `depends_on` strings to their full field names ({Dataset.id}.{field_name}). """ return [] def _get_root_datasets(self) -> dict[str, Dataset]: """Return a mapping from the id --> Dataset for all root datasets.""" if self.is_root: return {self.id: self} root_datasets = {} for source in self._sources: child_root_datasets = source._get_root_datasets() root_datasets = {**root_datasets, **child_root_datasets} return root_datasets def relax_types(self) -> None: """ Relax the types in this Dataset's schema and all upstream Datasets' schemas to be more permissive. """ # relax the types in this dataset's schema self._schema = relax_schema(self._schema) # relax the types in dataset's operator's input and output schemas self._operator.input_schema = None if self._operator.input_schema is None else relax_schema(self._operator.input_schema) self._operator.output_schema = relax_schema(self._operator.output_schema) # recursively relax the types in all upstream datasets for source in self._sources: source.relax_types() def get_upstream_datasets(self) -> list[Dataset]: """ Get the list of all upstream datasets that are sources to this dataset. """ # recursively get the upstream datasets upstream = [] for source in self._sources: upstream.extend(source.get_upstream_datasets()) upstream.append(source) return upstream def get_limit(self) -> int | None: """Get the limit applied to this Dataset, if any.""" if isinstance(self._operator, LimitScan): return self._operator.limit source_limits = [] for source in self._sources: source_limit = source.get_limit() if source_limit is not None: source_limits.append(source_limit) if len(source_limits) == 0: return None return min([limit for limit in source_limits if limit is not None]) def copy(self): return Dataset( sources=[source.copy() for source in self._sources], operator=self._operator.copy(), schema=self._schema, id=self.id, ) def join(self, other: Dataset, on: str | list[str], how: str = "inner") -> Dataset: """ Perform the specified join on the specified (list of) column(s) """ # enforce type for on if isinstance(on, str): on = [on] # construct new output schema combined_schema = union_schemas([self.schema, other.schema], join=True, on=on) # construct logical operator operator = JoinOp( input_schema=combined_schema, output_schema=combined_schema, condition="", on=on, how=how, depends_on=on, ) return Dataset(sources=[self, other], operator=operator, schema=combined_schema) def sem_join(self, other: Dataset, condition: str, desc: str | None = None, depends_on: str | list[str] | None = None, how: str = "inner") -> Dataset: """ Perform a semantic (inner) join on the specified join predicate """ # enforce type for depends_on if isinstance(depends_on, str): depends_on = [depends_on] # construct new output schema combined_schema = union_schemas([self.schema, other.schema], join=True) # construct logical operator operator = JoinOp( input_schema=combined_schema, output_schema=combined_schema, condition=condition, how=how, desc=desc, depends_on=depends_on, ) return Dataset(sources=[self, other], operator=operator, schema=combined_schema) def filter( self, filter: Callable, depends_on: str | list[str] | None = None, ) -> Dataset: """Add a user defined function as a filter to the Set. This filter will possibly restrict the items that are returned later.""" # construct Filter object f = None if callable(filter): f = Filter(filter_fn=filter) else: error_str = f"Only support callable for filter, currently got {type(filter)}" if isinstance(filter, str): error_str += ". Consider using sem_filter() for semantic filters." raise Exception(error_str) # enforce type for depends_on if isinstance(depends_on, str): depends_on = [depends_on] # construct logical operator operator = FilteredScan(input_schema=self.schema, output_schema=self.schema, filter=f, depends_on=depends_on) return Dataset(sources=[self], operator=operator, schema=self.schema) def sem_filter( self, filter: str, desc: str | None = None, depends_on: str | list[str] | None = None, ) -> Dataset: """Add a natural language description of a filter to the Set. This filter will possibly restrict the items that are returned later.""" # construct Filter object f = None if isinstance(filter, str): f = Filter(filter) else: raise Exception("sem_filter() only supports `str` input for _filter.", type(filter)) # enforce type for depends_on if isinstance(depends_on, str): depends_on = [depends_on] # construct logical operator operator = FilteredScan(input_schema=self.schema, output_schema=self.schema, filter=f, desc=desc, depends_on=depends_on) return Dataset(sources=[self], operator=operator, schema=self.schema) def _sem_map(self, cols: list[dict] | type[BaseModel] | None, cardinality: Cardinality, desc: str | None = None, depends_on: str | list[str] | None = None) -> Dataset: """Execute the semantic map operation with the appropriate cardinality.""" # construct new output schema new_output_schema = None if cols is None: new_output_schema = self.schema elif isinstance(cols, list): cols = create_schema_from_fields(cols) new_output_schema = union_schemas([self.schema, cols]) elif issubclass(cols, BaseModel): new_output_schema = union_schemas([self.schema, cols]) else: raise ValueError("`cols` must be a list of dictionaries or a BaseModel.") # enforce type for depends_on if isinstance(depends_on, str): depends_on = [depends_on] # construct logical operator operator = ConvertScan( input_schema=self.schema, output_schema=new_output_schema, cardinality=cardinality, udf=None, desc=desc, depends_on=depends_on, ) return Dataset(sources=[self], operator=operator, schema=new_output_schema) def sem_add_columns(self, cols: list[dict] | type[BaseModel], cardinality: Cardinality = Cardinality.ONE_TO_ONE, desc: str | None = None, depends_on: str | list[str] | None = None) -> Dataset: """ NOTE: we are renaming this function to `sem_map` and deprecating `sem_add_columns` in the next release of PZ. To update your code, simply change your calls from `.sem_add_columns(...)` to `.sem_map(...)`. The function arguments will stay the same. Add new columns by specifying the column names, descriptions, and types. The column will be computed during the execution of the Dataset. Example: sem_add_columns( [{'name': 'greeting', 'desc': 'The greeting message', 'type': str}, {'name': 'age', 'desc': 'The age of the person', 'type': int}, {'name': 'full_name', 'desc': 'The name of the person', 'type': str}] ) """ # issue deprecation warning warnings.warn( "we are renaming this function to `sem_map` and deprecating `sem_add_columns` in the next" " release of PZ. To update your code, simply change your calls from `.sem_add_columns(...)`" " to `.sem_map(...)`. The function arguments will stay the same.", DeprecationWarning, stacklevel=2 ) return self._sem_map(cols, cardinality, desc, depends_on) def sem_map(self, cols: list[dict] | type[BaseModel], desc: str | None = None, depends_on: str | list[str] | None = None) -> Dataset: """ Compute new field(s) by specifying their names, descriptions, and types. For each input there will be one output. The field(s) will be computed during the execution of the Dataset. Example: sem_map( [{'name': 'greeting', 'desc': 'The greeting message', 'type': str}, {'name': 'age', 'desc': 'The age of the person', 'type': int}, {'name': 'full_name', 'desc': 'The name of the person', 'type': str}] ) """ return self._sem_map(cols, Cardinality.ONE_TO_ONE, desc, depends_on) def sem_flat_map(self, cols: list[dict] | type[BaseModel], desc: str | None = None, depends_on: str | list[str] | None = None) -> Dataset: """ Compute new field(s) by specifying their names, descriptions, and types. For each input there will be one or more output(s). The field(s) will be computed during the execution of the Dataset. Example: sem_flat_map( cols=[ {'name': 'author_name', 'description': 'The name of the author', 'type': str}, {'name': 'institution', 'description': 'The institution of the author', 'type': str}, {'name': 'email', 'description': 'The author's email', 'type': str}, ] ) """ return self._sem_map(cols, Cardinality.ONE_TO_MANY, desc, depends_on) def _map(self, udf: Callable, cols: list[dict] | type[BaseModel] | None, cardinality: Cardinality, depends_on: str | list[str] | None = None) -> Dataset: """Execute the map operation with the appropriate cardinality.""" # construct new output schema new_output_schema = None if cols is None: new_output_schema = self.schema elif isinstance(cols, list): cols = create_schema_from_fields(cols) new_output_schema = union_schemas([self.schema, cols]) elif issubclass(cols, BaseModel): new_output_schema = union_schemas([self.schema, cols]) else: raise ValueError("`cols` must be a list of dictionaries, a BaseModel, or None.") # enforce type for depends_on if isinstance(depends_on, str): depends_on = [depends_on] # construct logical operator operator = ConvertScan( input_schema=self.schema, output_schema=new_output_schema, cardinality=cardinality, udf=udf, depends_on=depends_on, ) return Dataset(sources=[self], operator=operator, schema=new_output_schema) def add_columns(self, udf: Callable, cols: list[dict] | type[BaseModel] | None, cardinality: Cardinality = Cardinality.ONE_TO_ONE, depends_on: str | list[str] | None = None) -> Dataset: """ NOTE: we are renaming this function to `map` and deprecating `add_columns` in the next release of PZ. To update your code, simply change your calls from `.add_columns(...)` to `.map(...)`. The function arguments will stay the same. Compute new fields (or update existing ones) with a UDF. For each input, this function will compute one output. Set `cols=None` if your add_columns operation is not computing any new fields. Examples: add_columns( udf=compute_personal_greeting, cols=[ {'name': 'greeting', 'description': 'The greeting message', 'type': str}, {'name': 'age', 'description': 'The age of the person', 'type': int}, {'name': 'full_name', 'description': 'The name of the person', 'type': str}, ] ) """ # issue deprecation warning warnings.warn( "we are renaming this function to `map` and deprecating `add_columns` in the next" " release of PZ. To update your code, simply change your calls from `.add_columns(...)`" " to `.map(...)`. The function arguments will stay the same.", DeprecationWarning, stacklevel=2 ) # sanity check inputs if udf is None: raise ValueError("`udf` must be provided for add_columns.") return self._map(udf, cols, cardinality, depends_on) def map(self, udf: Callable, cols: list[dict] | type[BaseModel] | None, depends_on: str | list[str] | None = None) -> Dataset: """ Compute new fields (or update existing ones) with a UDF. For each input, this function will compute one output. Set `cols=None` if your map is not computing any new fields. Examples: map( udf=compute_personal_greeting, cols=[ {'name': 'greeting', 'description': 'The greeting message', 'type': str}, {'name': 'age', 'description': 'The age of the person', 'type': int}, {'name': 'full_name', 'description': 'The name of the person', 'type': str}, ] ) """ # sanity check inputs if udf is None: raise ValueError("`udf` must be provided for map.") return self._map(udf, cols, Cardinality.ONE_TO_ONE, depends_on) def flat_map(self, udf: Callable, cols: list[dict] | type[BaseModel] | None, depends_on: str | list[str] | None = None) -> Dataset: """ Compute new fields (or update existing ones) with a UDF. For each input, this function will compute one or more outputs. Set `cols=None` if your flat_map is not computing any new fields. Examples: flat_map( udf=extract_paper_authors, cols=[ {'name': 'author_name', 'description': 'The name of the author', 'type': str}, {'name': 'institution', 'description': 'The institution of the author', 'type': str}, {'name': 'email', 'description': 'The author's email', 'type': str}, ] ) """ # sanity check inputs if udf is None: raise ValueError("`udf` must be provided for map.") return self._map(udf, cols, Cardinality.ONE_TO_MANY, depends_on) def count(self) -> Dataset: """Apply a count aggregation to this set""" operator = Aggregate(input_schema=self.schema, agg_func=AggFunc.COUNT) return Dataset(sources=[self], operator=operator, schema=operator.output_schema) def average(self) -> Dataset: """Apply an average aggregation to this set""" operator = Aggregate(input_schema=self.schema, agg_func=AggFunc.AVERAGE) return Dataset(sources=[self], operator=operator, schema=operator.output_schema) def sum(self) -> Dataset: """Apply a summation to this set""" operator = Aggregate(input_schema=self.schema, agg_func=AggFunc.SUM) return Dataset(sources=[self], operator=operator, schema=operator.output_schema) def min(self) -> Dataset: """Apply an min operator to this set""" operator = Aggregate(input_schema=self.schema, agg_func=AggFunc.MIN) return Dataset(sources=[self], operator=operator, schema=operator.output_schema) def max(self) -> Dataset: """Apply an max operator to this set""" operator = Aggregate(input_schema=self.schema, agg_func=AggFunc.MAX) return Dataset(sources=[self], operator=operator, schema=operator.output_schema) def groupby(self, groupby: GroupBySig) -> Dataset: output_schema = groupby.output_schema() operator = GroupByAggregate(input_schema=self.schema, output_schema=output_schema, group_by_sig=groupby) return Dataset(sources=[self], operator=operator, schema=output_schema) def sem_agg(self, col: dict | type[BaseModel], agg: str, depends_on: str | list[str] | None = None) -> Dataset: """ Apply a semantic aggregation to this set. The `agg` string will be applied using an LLM over the entire set of inputs' fields specified in `depends_on` to generate the output `col`. Example: sem_agg( col={'name': 'overall_sentiment', 'desc': 'The overall sentiment of the reviews', 'type': str}, agg="Compute the overall sentiment of the reviews as POSITIVE or NEGATIVE.", depends_on="review_text", ) """ # construct new output schema new_output_schema = None if isinstance(col, dict): new_output_schema = create_schema_from_fields([col]) elif issubclass(col, BaseModel): assert len(col.model_fields) == 1, "For semantic aggregation, when passing a BaseModel to `col` it must have exactly one field." new_output_schema = col else: raise ValueError("`col` must be a dictionary or a single-field BaseModel.") # enforce type for depends_on if isinstance(depends_on, str): depends_on = [depends_on] # construct logical operator operator = Aggregate(input_schema=self.schema, output_schema=new_output_schema, agg_str=agg, depends_on=depends_on) return Dataset(sources=[self], operator=operator, schema=operator.output_schema) def sem_topk( self, index: Collection, search_attr: str, output_attrs: list[dict] | type[BaseModel], search_func: Callable | None = None, k: int = -1, ) -> Dataset: """ Retrieve the top-k nearest neighbors of the value of the `search_attr` from the `index` and use these results to construct the `output_attrs` field(s). """ # construct new output schema new_output_schema = None if isinstance(output_attrs, list): output_attrs = create_schema_from_fields(output_attrs) new_output_schema = union_schemas([self.schema, output_attrs]) elif issubclass(output_attrs, BaseModel): new_output_schema = union_schemas([self.schema, output_attrs]) else: raise ValueError("`output_attrs` must be a list of dictionaries or a BaseModel.") # TODO: revisit once we can think through abstraction(s) # # construct the PZIndex from the user-provided index # index = index_factory(index) # construct logical operator operator = TopKScan( input_schema=self.schema, output_schema=new_output_schema, index=index, search_func=search_func, search_attr=search_attr, output_attrs=output_attrs, k=k, ) return Dataset(sources=[self], operator=operator, schema=new_output_schema) def limit(self, n: int) -> Dataset: """Limit the set size to no more than n rows""" operator = LimitScan(input_schema=self.schema, output_schema=self.schema, limit=n) return Dataset(sources=[self], operator=operator, schema=self.schema) def distinct(self, distinct_cols: list[str] | None = None) -> Dataset: """Return a new Dataset with distinct rows based on the current schema.""" operator = Distinct(input_schema=self.schema, output_schema=self.schema, distinct_cols=distinct_cols) return Dataset(sources=[self], operator=operator, schema=self.schema) def project(self, project_cols: list[str] | str) -> Dataset: """Project the Set to only include the specified columns.""" project_cols = project_cols if isinstance(project_cols, list) else [project_cols] new_output_schema = project(self.schema, project_cols) operator = Project(input_schema=self.schema, output_schema=new_output_schema, project_cols=project_cols) return Dataset(sources=[self], operator=operator, schema=new_output_schema) def run(self, config: QueryProcessorConfig | None = None, **kwargs): """Invoke the QueryProcessor to execute the query. `kwargs` will be applied to the QueryProcessorConfig.""" # TODO: this import currently needs to be here to avoid a circular import; we should fix this in a subsequent PR from palimpzest.query.processor.query_processor_factory import QueryProcessorFactory # as syntactic sugar, we will allow some keyword arguments to parameterize our policies policy = construct_policy_from_kwargs(**kwargs) if policy is not None: kwargs["policy"] = policy # construct unique logical op ids for all operators in this dataset self._generate_unique_logical_op_ids() return QueryProcessorFactory.create_and_run_processor(self, config) def optimize_and_run(self, config: QueryProcessorConfig | None = None, train_dataset: dict[str, Dataset] | Dataset | None = None, validator: Validator | None = None, **kwargs): """Optimize the PZ program using the train_dataset and validator before running the optimized plan.""" # TODO: this import currently needs to be here to avoid a circular import; we should fix this in a subsequent PR from palimpzest.query.processor.query_processor_factory import QueryProcessorFactory # confirm that either train_dataset or validator is provided assert train_dataset is not None or validator is not None, "Must provide at least one of train_dataset or validator to use optimize_and_run()" # validate the train_dataset has one input for each source dataset and normalize its type to be a dict if train_dataset is not None: root_datasets = self._get_root_datasets() if isinstance(train_dataset, Dataset) and len(root_datasets) > 1: raise ValueError( "For plans with more than one root dataset, `train_dataset` must be a dictionary mapping" " {'dataset_id' --> Dataset} for all root Datasets" ) elif isinstance(train_dataset, Dataset): root_dataset_id = list(root_datasets.values())[0].id if train_dataset.id != root_dataset_id: warnings.warn( f"train_dataset.id={train_dataset.id} does not match root dataset id={root_dataset_id}\n" f"Setting train_dataset to be the training data for root dataset with id={root_dataset_id} anyways.", stacklevel=2, ) train_dataset = {root_dataset_id: train_dataset} elif not all(dataset_id in train_dataset for dataset_id in root_datasets): missing_ids = [dataset_id for dataset_id in root_datasets if dataset_id not in train_dataset] raise ValueError( f"`train_dataset` is missing the following root dataset id(s): {missing_ids}" ) # as syntactic sugar, we will allow some keyword arguments to parameterize our policies policy = construct_policy_from_kwargs(**kwargs) if policy is not None: kwargs["policy"] = policy config.policy = policy # construct unique logical op ids for all operators in this dataset self._generate_unique_logical_op_ids() return QueryProcessorFactory.create_and_run_processor(self, config, train_dataset, validator) ================================================ FILE: src/palimpzest/core/data/index_dataset.py ================================================ from __future__ import annotations from abc import ABC, abstractmethod from chromadb.api.models.Collection import Collection def index_factory(index: Collection) -> PZIndex: """ Factory function to create a PZ index based on the type of the provided index. Args: index (Collection): The index provided by the user. Returns: PZIndex: The PZ wrapped Index. """ if isinstance(index, Collection): return ChromaIndex(index) else: raise TypeError(f"Unsupported index type: {type(index)}\nindex must be a `chromadb.api.models.Collection.Collection`") class BaseIndex(ABC): def __init__(self, index: Collection): self.index = index def __str__(self): """ Return a string representation of the index. """ return f"{self.__class__.__name__}" @abstractmethod def search(self, query_embedding: list[float] | list[list[float]], results_per_query: int = 1) -> list | list[list]: """ Query the index with a string or a list of strings. Args: query (str | list[str]): The query string or list of strings to search for. results_per_query (int): The number of top results to retrieve for each query. Returns: list | list[list]: The top results for the query. If query is a list, the top results for each query in the list are returned. Each list will contain the raw elements yielded by the index. This way, users can program against the results they expect to get from e.g. chromadb or ragatouille. """ pass class ChromaIndex(BaseIndex): def __init__(self, index: Collection): assert isinstance(index, Collection), "ChromaIndex input must be a `chromadb.api.models.Collection.Collection`" super().__init__(index) # define type for PZIndex PZIndex = ChromaIndex ================================================ FILE: src/palimpzest/core/data/iter_dataset.py ================================================ from __future__ import annotations import base64 import os from abc import ABC, abstractmethod from io import BytesIO from pathlib import Path import pandas as pd from bs4 import BeautifulSoup from pydantic import BaseModel from palimpzest import constants from palimpzest.core.data import dataset from palimpzest.core.lib.schemas import ( AudioFile, DefaultSchema, ImageFile, PDFFile, TextFile, WebPage, XLSFile, create_schema_from_df, create_schema_from_fields, ) from palimpzest.query.operators.logical import BaseScan from palimpzest.tools.pdfparser import get_text_from_pdf #################### ### BASE CLASSES ### #################### class IterDataset(dataset.Dataset, ABC): """ The `IterDataset` is an abstract base class for root `Datasets` whose data is accessed via iteration. Classes which inherit from this class must implement two methods: - `__len__()`: which returns the number of elements in the dataset - `__getitem__(idx: int)`: which takes in an `idx` and returns the element at that index """ def __init__(self, id: str, schema: type[BaseModel] | list[dict]) -> None: """ Constructor for the `IterDataset` class. Args: id (str): a string identifier for the `Dataset` schema (BaseModel | list[dict]): The output schema of the records returned by the `Dataset` """ # compute Schema and call parent constructor schema = create_schema_from_fields(schema) if isinstance(schema, list) else schema super().__init__(sources=None, operator=BaseScan(datasource=self, output_schema=schema), schema=schema, id=id) @abstractmethod def __len__(self) -> int: """Returns the number of items in the `Dataset`.""" pass @abstractmethod def __getitem__(self, idx: int) -> dict: """ Returns a single item from the `Dataset` at the given index. Args: idx (int): The index of the item to return Returns: dict: A dictionary representing the item at the given index. The dictionary keys (i.e. fields) should match the fields specified in the schema of the dataset, and the values should be the values associated with those fields. # Example return value {"field1": value1, "field2": value2, ...} """ pass class BaseFileDataset(IterDataset): """ BaseFileDataset is the base class for multiple `IterDatasets` which iterate over different types of files. """ def __init__(self, path: str, **kwargs) -> None: """ Constructor for the `BaseFileDataset` class. Args: path (str): The path to the file kwargs (dict): Keyword arguments containing the `Dataset's` id and file-specific `Schema` """ # check that path is a valid file or directory assert os.path.isfile(path) or os.path.isdir(path), f"Path {path} is not a file nor a directory" # get list of filepaths self.filepaths = [] if os.path.isfile(path): self.filepaths = [path] else: self.filepaths = [ os.path.join(path, filename) for filename in sorted(os.listdir(path)) if os.path.isfile(os.path.join(path, filename)) ] # call parent constructor to set id, operator, and schema super().__init__(**kwargs) def __len__(self) -> int: return len(self.filepaths) class BaseFileDirectoryDataset(IterDataset): """ BaseFileDirectoryDataset is the base class for multiple `IterDatasets` which iterate over different types of files. This class walks the entire directory tree rooted at `path`. """ def __init__(self, path: str, **kwargs) -> None: """ Constructor for the `BaseFileDataset` class. Args: path (str): The path to the file kwargs (dict): Keyword arguments containing the `Dataset's` id and file-specific `Schema` """ # check that path is a valid file or directory assert os.path.isfile(path) or os.path.isdir(path), f"Path {path} is not a file nor a directory" # get list of filepaths self.filepaths = [] if os.path.isfile(path): self.filepaths = [path] else: self.filepaths = [] for root, _, files in os.walk(path): for file in files: fp = os.path.join(root, file) self.filepaths.append(fp) self.filepaths = sorted(self.filepaths) # call parent constructor to set id, operator, and schema super().__init__(**kwargs) def __len__(self) -> int: return len(self.filepaths) ######################## ### CONCRETE CLASSES ### ######################## class MemoryDataset(IterDataset): """ MemoryDataset returns one or more dictionaries that reflect the contents of an in-memory Python object `vals`. If `vals` is not a pd.DataFrame, then the dictionary returned by `__getitem__()` has a single field called "value". Otherwise, the dictionary contains the key-value mapping from columns to values for the `idx` row in the dataframe. TODO(gerardo): Add support for other types of in-memory data structures (he has some code for subclassing MemoryDataset on his branch) """ def __init__(self, id: str, vals: list | pd.DataFrame, schema: type[BaseModel] | list[dict] | None = None) -> None: """ Constructor for the `MemoryDataset` class. The `schema` is set to the default `DefaultSchema` schema. If `vals` is a pd.DataFrame, then the schema is set to the schema inferred from the DataFrame. Args: id (str): a string identifier for the `Dataset` vals (Any): The in-memory data to iterate over """ # if list[dict] --> convert to pd.DataFrame first self.vals = pd.DataFrame(vals) if isinstance(vals, list) and all([isinstance(item, dict) for item in vals]) else vals if schema is None: schema = create_schema_from_df(self.vals) if isinstance(self.vals, pd.DataFrame) else DefaultSchema super().__init__(id=id, schema=schema) def __len__(self) -> int: return len(self.vals) def __getitem__(self, idx: int) -> dict: """ Returns a dictionary with the value(s) for the element at the specified `idx` in `vals`. Args: idx (int): The index of the item to return Returns: dict: If `vals` is not a pd.DataFrame, then the dictionary has a single field called "value". Otherwise, the dictionary contains the key-value mapping from columns to values for the `idx` row in the dataframe. .. code-block:: python # Example return value at idx = 0, for the following list of values # [42, 43, 44, ...] {"value": 42} # Example return value at idx = 0, for the following DataFrame: # +---------+---------+---------+ # | name | job | hobby | # +---------+---------+---------+ # | Alice | doctor | tennis | # | Bob | lawyer | chess | # +---------+---------+---------+ {"name": "Alice", "job": "doctor", "hobby": "tennis"} """ item = ( self.vals.iloc[idx].to_dict() if isinstance(self.vals, pd.DataFrame) else {"value": self.vals[idx]} ) return item class HTMLFileDataset(BaseFileDataset): """ HTMLFileDataset returns a dictionary for each HTML file in a directory. Each dictionary contains the filename, raw HTML content, and parsed content of a single HTML file in the directory. """ def __init__(self, id: str, path: str) -> None: """ Constructor for the `HTMLFileDataset` class. The `schema` is set to the `WebPage` schema. Args: id (str): a string identifier for the `Dataset` path (str): The path to the directory """ super().__init__(path=path, id=id, schema=WebPage) self.filepaths = [fp for fp in self.filepaths if fp.endswith(tuple(constants.HTML_EXTENSIONS))] def _html_to_text_with_links(self, html: str) -> str: # Parse the HTML content soup = BeautifulSoup(html, "html.parser") # Find all hyperlink tags for a in soup.find_all("a"): # Check if the hyperlink tag has an 'href' attribute if a.has_attr("href"): # Replace the hyperlink with its text and URL in parentheses a.replace_with(f"{a.text} ({a['href']})") # Extract text from the modified HTML text = soup.get_text(separator="\n", strip=True) return text def __getitem__(self, idx: int) -> dict: """ Returns a dictionary with the filename, raw HTML content, and parsed content of the HTML file at the specified `idx`. Args: idx (int): The index of the item to return Returns: dict: A dictionary with the filename, raw HTML content, and parsed content of the HTML file. .. code-block:: python { "filename": "file.html", "html": "raw HTML content here", "text": "parsed text content here", } """ item = {} filepath = self.filepaths[idx] item["filename"] = os.path.basename(filepath) with open(filepath) as f: text_content = f.read() html = text_content tokens = html.split()[: constants.MAX_HTML_ROWS] item["html"] = " ".join(tokens) stripped_html = self._html_to_text_with_links(text_content) tokens = stripped_html.split()[: constants.MAX_HTML_ROWS] item["text"] = " ".join(tokens) return item class ImageFileDataset(BaseFileDataset): """ ImageFileDataset returns a dictionary for each image file in a directory. Each dictionary contains the filename and the base64 encoded bytes content of a single image file in the directory. """ def __init__(self, id: str, path: str) -> None: """ Constructor for the `ImageFileDataset` class. The `schema` is set to the `ImageFile` schema. Args: id (str): a string identifier for the `Dataset` path (str): The path to the directory """ super().__init__(path=path, id=id, schema=ImageFile) self.filepaths = [fp for fp in self.filepaths if fp.endswith(tuple(constants.IMAGE_EXTENSIONS))] def __getitem__(self, idx: int) -> dict: """ Returns a dictionary with the filename and base64 encoded bytes content of the image file at the specified `idx`. Args: idx (int): The index of the item to return Returns: dict: A dictionary with the filename and base64 encoded bytes content of the image file. .. code-block:: python { "filename": "image.jpg", "contents": b"base64 encoded image content here", } """ filepath = self.filepaths[idx] filename = os.path.basename(filepath) with open(filepath, "rb") as f: contents = base64.b64encode(f.read()).decode("utf-8") return {"filename": filename, "contents": contents} class PDFFileDataset(BaseFileDataset): """ PDFFileDataset returns a dictionary for each PDF file in a directory. Each dictionary contains the filename, raw PDF content, and parsed text content of a single PDF file in the directory. This class also uses one of a predefined set of PDF processors to extract text content from the PDF files. """ def __init__( self, id: str, path: str, pdfprocessor: str = "pypdf", file_cache_dir: str = "/tmp", ) -> None: """ Constructor for the `PDFFileDataset` class. The `schema` is set to the `PDFFile` schema. Args: id (str): a string identifier for the `Dataset` path (str): The path to the directory pdfprocessor (str): The PDF processor to use for extracting text content from the PDF files file_cache_dir (str): The directory to store the temporary files generated during PDF processing """ super().__init__(path=path, id=id, schema=PDFFile) self.filepaths = [fp for fp in self.filepaths if fp.endswith(tuple(constants.PDF_EXTENSIONS))] self.pdfprocessor = pdfprocessor self.file_cache_dir = file_cache_dir def __getitem__(self, idx: int) -> dict: """ Returns a dictionary with the filename, raw PDF content, and parsed text content of the PDF file at the specified `idx`. Args: idx (int): The index of the item to return Returns: dict: A dictionary with the filename, raw PDF content, and parsed text content of the PDF file. .. code-block:: python { "filename": "file.pdf", "contents": b"raw PDF content here", "text_contents": "parsed text content here", } """ filepath = self.filepaths[idx] pdf_filename = os.path.basename(filepath) with open(filepath, "rb") as f: pdf_bytes = f.read() # generate text_content from PDF text_content = get_text_from_pdf(pdf_filename, pdf_bytes, pdfprocessor=self.pdfprocessor, file_cache_dir=self.file_cache_dir) # construct and return item return {"filename": pdf_filename, "contents": pdf_bytes, "text_contents": text_content} class TextFileDataset(BaseFileDataset): """ TextFileDataset returns a dictionary for each text file in a directory. Each dictionary contains the filename and contents of a single text file in the directory. """ def __init__(self, id: str, path: str) -> None: """ Constructor for the `TextFileDataset` class. The `schema` is set to the `TextFile` schema. Args: id (str): a string identifier for the `Dataset` path (str): The path to the directory """ super().__init__(path=path, id=id, schema=TextFile) def __getitem__(self, idx: int) -> dict: """ Returns a dictionary with the filename and contents of the text file at the specified `idx`. Args: idx (int): The index of the item to return Returns: dict: A dictionary with the filename and contents of the text file. .. code-block:: python { "filename": "file.txt", "contents": "text content here", } """ filepath = self.filepaths[idx] filename = os.path.basename(filepath) with open(filepath) as f: contents = f.read() return {"filename": filename, "contents": contents} class XLSFileDataset(BaseFileDataset): """ XLSFileDataset returns a dictionary for each XLS file in a directory. Each dictionary contains the filename, contents, sheet names, and the number of sheets for a single XLS file in the directory. """ def __init__(self, id: str, path: str) -> None: """ Constructor for the `XLSFileDataset` class. The `schema` is set to the `XLSFile` schema. """ super().__init__(path=path, id=id, schema=XLSFile) self.filepaths = [fp for fp in self.filepaths if fp.endswith(tuple(constants.XLS_EXTENSIONS))] def __getitem__(self, idx: int) -> dict: """ Returns a dictionary with the filename, contents, sheet names, and the number of sheets of the XLS file at the specified `idx`. Args: idx (int): The index of the item to return Returns: dict: A dictionary with the filename, contents, sheet names, and the number of sheets of the XLS file. .. code-block:: python { "filename": "file.xls", "contents": b"raw XLS content here", "sheet_names": ["Sheet1", "Sheet2", "Sheet3], "number_sheets": 3, } """ filepath = self.filepaths[idx] filename = os.path.basename(filepath) with open(filepath, "rb") as f: contents = f.read() xls = pd.ExcelFile(BytesIO(contents), engine="openpyxl") return { "filename": filename, "contents": contents, "sheet_names": xls.sheet_names, "number_sheets": len(xls.sheet_names), } class AudioFileDataset(BaseFileDirectoryDataset): """ AudioFileDataset returns a dictionary for each audio file in a directory. Each dictionary contains the filename and the base64 encoded bytes content of a single audio file in the directory. """ def __init__(self, id: str, path: str) -> None: """ Constructor for the `AudioFileDataset` class. The `schema` is set to the `AudioFile` schema. Args: id (str): a string identifier for the `Dataset` path (str): The path to the directory """ super().__init__(path=path, id=id, schema=AudioFile) self.filepaths = [fp for fp in self.filepaths if fp.endswith(tuple(constants.AUDIO_EXTENSIONS))] def __getitem__(self, idx: int) -> dict: """ Returns a dictionary with the filename and base64 encoded bytes content of the audio file at the specified `idx`. Args: idx (int): The index of the item to return Returns: dict: A dictionary with the filename and base64 encoded bytes content of the audio file. .. code-block:: python { "filename": "audio.wav", "contents": b"base64 encoded audio content here", } """ filepath = self.filepaths[idx] filename = os.path.basename(filepath) with open(filepath, "rb") as f: contents = base64.b64encode(f.read()).decode("utf-8") return {"filename": filename, "contents": contents} def get_local_source(id: str, path: str | Path, **kwargs) -> dataset.Dataset: """Return a `Dataset` for a local file or directory.""" if os.path.isfile(path): return TextFileDataset(id, path) elif os.path.isdir(path): if all([f.endswith(tuple(constants.IMAGE_EXTENSIONS)) for f in os.listdir(path)]): return ImageFileDataset(id, path) elif all([f.endswith(tuple(constants.PDF_EXTENSIONS)) for f in os.listdir(path)]): pdfprocessor = kwargs.get("pdfprocessor", constants.DEFAULT_PDF_PROCESSOR) file_cache_dir = kwargs.get("file_cache_dir", "/tmp") return PDFFileDataset( id=id, path=path, pdfprocessor=pdfprocessor, file_cache_dir=file_cache_dir ) elif all([f.endswith(tuple(constants.XLS_EXTENSIONS)) for f in os.listdir(path)]): return XLSFileDataset(id, path) elif all([f.endswith(tuple(constants.HTML_EXTENSIONS)) for f in os.listdir(path)]): return HTMLFileDataset(id, path) else: return TextFileDataset(id, path) else: raise ValueError(f"Path {path} is invalid. Does not point to a file or directory.") def resolve_datasource(id: str, source: str | Path | list | pd.DataFrame, **kwargs) -> dataset.Dataset: """ This helper function returns a `Dataset` object based on the `source` type. The returned `Dataset` object is guaranteed to have a schema. """ if isinstance(source, (str, Path)): source = get_local_source(id, source, **kwargs) elif isinstance(source, (list, pd.DataFrame)): source = MemoryDataset(id=id, vals=source) else: raise ValueError(f"Invalid source type: {type(source)}, We only support str, Path, list[dict], and pd.DataFrame") return source ================================================ FILE: src/palimpzest/core/elements/__init__.py ================================================ ================================================ FILE: src/palimpzest/core/elements/filters.py ================================================ from __future__ import annotations from typing import Any, Callable ############################# # Filters that can be applied against a particular Schema ############################# # TODO: think through a way to give filter functions fixed strings that could not be affected by a copy # potentially changing the address of a function; I don't think this happens today, but it's worth safeguarding against class Filter: """A filter that can be applied to a Set""" def __init__(self, filter_condition: str | None = None, filter_fn: Callable | None = None) -> None: self.filter_condition = filter_condition self.filter_fn = filter_fn def serialize(self) -> dict[str, Any]: return { "filter_condition": self.filter_condition, "filter_fn": self.filter_fn.__name__ if self.filter_fn is not None else None, } def get_filter_str(self) -> str: return self.filter_condition if self.filter_condition is not None else self.filter_fn.__name__ def __repr__(self) -> str: return "Filter(" + self.get_filter_str() + ")" def __hash__(self) -> int: # custom hash function return hash(self.filter_condition) if self.filter_condition is not None else hash(self.filter_fn.__name__) def __eq__(self, other) -> bool: # __eq__ should be defined for consistency with __hash__ return ( isinstance(other, Filter) and self.filter_condition == other.filter_condition and self.filter_fn == other.filter_fn ) def __str__(self) -> str: return self.get_filter_str() ================================================ FILE: src/palimpzest/core/elements/groupbysig.py ================================================ from __future__ import annotations from typing import Any from pydantic import BaseModel from palimpzest.core.lib.schemas import create_schema_from_fields # TODO: # - move the arguments for group_by_fields, agg_funcs, and agg_fields into the Dataset.groupby() operator # - construct the correct output schema using the input schema and the group by and aggregation fields # - remove/update all other references to GroupBySig in the codebase # TODO: # - move the arguments for group_by_fields, agg_funcs, and agg_fields into the Dataset.groupby() operator # - construct the correct output schema using the input schema and the group by and aggregation fields # - remove/update all other references to GroupBySig in the codebase # signature for a group by aggregate that applies # group and aggregation to an input tuple class GroupBySig: def __init__(self, group_by_fields: list[str], agg_funcs: list[str], agg_fields: list[str]): self.group_by_fields = group_by_fields self.agg_funcs = agg_funcs self.agg_fields = agg_fields def validate_schema(self, input_schema: type[BaseModel]) -> tuple[bool, str | None]: for f in self.group_by_fields: if f not in input_schema.model_fields: return (False, "Supplied schema has no field " + f) for f in self.agg_fields: if f not in input_schema.model_fields: return (False, "Supplied schema has no field " + f) return (True, None) def serialize(self) -> dict[str, Any]: out = { "group_by_fields": self.group_by_fields, "agg_funcs": self.agg_funcs, "agg_fields": self.agg_fields, } return out def __str__(self) -> str: return "GroupBy(" + repr(self.serialize()) + ")" def __hash__(self) -> int: # custom hash function return hash(repr(self.serialize())) def __eq__(self, other) -> bool: # __eq__ should be defined for consistency with __hash__ return isinstance(other, GroupBySig) and self.serialize() == other.serialize() def get_agg_field_names(self) -> list[str]: ops = [] for i in range(0, len(self.agg_fields)): ops.append(self.agg_funcs[i] + "(" + self.agg_fields[i] + ")") return ops # TODO: output schema needs to account for input schema types and create new output schema types def output_schema(self) -> type[BaseModel]: # the output class varies depending on the group by, so here # we dynamically construct this output fields = [] for g in self.group_by_fields: f = {"name": g, "type": Any, "desc": f"Group by field: {g}"} fields.append(f) ops = self.get_agg_field_names() for op in ops: f = {"name": op, "type": Any, "desc": f"Aggregate field: {op}"} fields.append(f) return create_schema_from_fields(fields) ================================================ FILE: src/palimpzest/core/elements/records.py ================================================ from __future__ import annotations import json from collections.abc import Generator from copy import deepcopy from typing import Any import pandas as pd from pydantic import BaseModel from pydantic.fields import FieldInfo from palimpzest.core.data import context from palimpzest.core.lib.schemas import ( AUDIO_FIELD_TYPES, IMAGE_FIELD_TYPES, AudioBase64, AudioFilepath, ImageBase64, ImageFilepath, ImageURL, project, union_schemas, ) from palimpzest.core.models import ExecutionStats, PlanStats, RecordOpStats from palimpzest.utils.hash_helpers import hash_for_id class DataRecord: """A DataRecord is a single record of data matching some schema defined by a BaseModel.""" def __init__( self, data_item: BaseModel, source_indices: str | int | list[str | int], parent_ids: str | list[str] | None = None, cardinality_idx: int | None = None, ): # check that source_indices are provided assert source_indices is not None, "Every DataRecord must be constructed with source index (or indices)" # normalize to list[str] if not isinstance(source_indices, list): source_indices = [source_indices] # normalize to list[str] if isinstance(parent_ids, str): parent_ids = [parent_ids] # data for the data record self._data_item = data_item # the index in the root Dataset from which this DataRecord is derived; # each source index takes the form: f"{root_dataset.id}-{idx}" self._source_indices = sorted(source_indices) # the id(s) of the parent record(s) from which this DataRecord is derived self._parent_ids = parent_ids # store the cardinality index self._cardinality_idx = cardinality_idx # indicator variable which may be flipped by filter operations to signal when a record has been filtered out self._passed_operator = True # NOTE: Record ids are hashed based on: # 0. their schema (keys) # 1. their parent record id(s) (or source_indices if there is no parent record) # 2. their index in the fan out (if this is in a one-to-many operation) # # We currently do NOT hash just based on record content (i.e. schema (key, value) pairs) # because multiple outputs for a given operation may have the exact same # schema (key, value) pairs. # # We may revisit this hashing scheme in the future. # unique identifier for the record schema_fields = sorted(list(type(data_item).model_fields)) id_str = ( str(schema_fields) + str(parent_ids) if parent_ids is not None else str(self._source_indices) if cardinality_idx is None else str(schema_fields) + str(cardinality_idx) + str(parent_ids) if parent_ids is not None else str(self._source_indices) ) self._id = hash_for_id(id_str) # TODO: raise an exception if one of these fields is present in the schema # - put these in a constant list up top # - import the constant list in Dataset (if possible) and check at plan creation time def __setattr__(self, name: str, value: Any, /) -> None: if name in ["_data_item", "_source_indices", "_parent_ids", "_cardinality_idx", "_passed_operator", "_id"]: super().__setattr__(name, value) else: setattr(self._data_item, name, value) def __getattr__(self, name: str) -> Any: return getattr(self._data_item, name) def __getitem__(self, field: str) -> Any: return getattr(self._data_item, field) def __setitem__(self, field: str, value: Any) -> None: setattr(self._data_item, field, value) def __str__(self, truncate: int | None = 15) -> str: if truncate is not None: items = (f"{k}={str(v)[:truncate]!r}{'...' if len(str(v)) > truncate else ''}" for k, v in sorted(self._data_item.model_dump().items())) else: items = (f"{k}={v!r}" for k, v in sorted(self._data_item.model_dump().items())) return "{}({})".format(type(self).__name__, ", ".join(items)) def __repr__(self) -> str: return self.__str__(truncate=None) def __eq__(self, other): return isinstance(other, DataRecord) and self._data_item == other._data_item def __hash__(self): return hash(self.to_json_str(bytes_to_str=True, sorted=True)) def __iter__(self): yield from self._data_item.__iter__() def get_field_names(self): return list(type(self._data_item).model_fields.keys()) def get_field_type(self, field_name: str) -> FieldInfo: return type(self._data_item).model_fields[field_name] @property def schema(self) -> type[BaseModel]: return type(self._data_item) def copy(self) -> DataRecord: # get the set of fields to copy from the parent record copy_field_names = [field.split(".")[-1] for field in self.get_field_names()] # copy field types and values from the parent data_item = {field_name: self[field_name] for field_name in copy_field_names} # make copy of the current record new_dr = DataRecord( self.schema(**data_item), source_indices=self._source_indices, parent_ids=self._parent_ids, cardinality_idx=self._cardinality_idx, ) # copy the passed_operator attribute new_dr._passed_operator = self._passed_operator return new_dr @staticmethod def from_parent( schema: type[BaseModel], data_item: dict, parent_record: DataRecord, project_cols: list[str] | None = None, cardinality_idx: int | None = None, ) -> DataRecord: # if project_cols is None, then the new schema is a union of the provided schema and parent_record.schema; # if project_cols is an empty list, then the new schema is simply the provided schema # otherwise, it's a ProjectSchema new_schema = None if project_cols is None: new_schema = union_schemas([schema, parent_record.schema]) elif project_cols == []: new_schema = schema else: new_schema = union_schemas([schema, parent_record.schema]) new_schema = project(new_schema, project_cols) # get the set of fields and field descriptions to copy from the parent record copy_field_names = parent_record.get_field_names() if project_cols is None else project_cols copy_field_names = [field.split(".")[-1] for field in copy_field_names] # copy fields from the parent data_item.update({field_name: parent_record[field_name] for field_name in copy_field_names}) # corner-case: wrap values in lists if the new schema expects a list but the data item has a single value for field_name, field_info in new_schema.model_fields.items(): field_should_be_list = hasattr(field_info.annotation, '__origin__') and field_info.annotation.__origin__ is list field_is_not_list = field_name in data_item and not isinstance(data_item[field_name], list) if field_should_be_list and field_is_not_list: data_item[field_name] = [data_item[field_name]] # make new record which has parent_record as its parent (and the same source_indices) new_dr = DataRecord( new_schema(**data_item), source_indices=parent_record._source_indices, parent_ids=[parent_record._id], cardinality_idx=cardinality_idx, ) return new_dr @staticmethod def from_agg_parents( data_item: BaseModel, parent_records: DataRecordSet, cardinality_idx: int | None = None, ) -> DataRecord: # flatten source indices from all parents source_indices = [ source_idx for parent_record in parent_records for source_idx in parent_record._source_indices ] # make new record which has all parent records as its parents return DataRecord( data_item, source_indices=source_indices, parent_ids=[parent_record._id for parent_record in parent_records], cardinality_idx=cardinality_idx, ) @staticmethod def from_join_parents( schema: type[BaseModel], left_parent_record: DataRecord | None, right_parent_record: DataRecord | None, project_cols: list[str] | None = None, cardinality_idx: int = None, ) -> DataRecord: # get the set of fields and field descriptions to copy from the parent record(s) left_copy_field_names = [] if left_parent_record is None else ( left_parent_record.get_field_names() if project_cols is None else [col for col in project_cols if col in left_parent_record.get_field_names()] ) right_copy_field_names = [] if right_parent_record is None else ( right_parent_record.get_field_names() if project_cols is None else [col for col in project_cols if col in right_parent_record.get_field_names()] ) left_copy_field_names = [field.split(".")[-1] for field in left_copy_field_names] right_copy_field_names = [field.split(".")[-1] for field in right_copy_field_names] # copy fields from the parents data_item = {field_name: left_parent_record[field_name] for field_name in left_copy_field_names} for field_name in right_copy_field_names: new_field_name = field_name if field_name in left_copy_field_names: new_field_name = f"{field_name}_right" data_item[new_field_name] = right_parent_record[field_name] # for any missing fields in the schema, set them to None for field_name in schema.model_fields: if field_name not in data_item: data_item[field_name] = None # make new record which has left and right parent record as its parents left_parent_source_indices = [] if left_parent_record is None else list(left_parent_record._source_indices) right_parent_source_indices = [] if right_parent_record is None else list(right_parent_record._source_indices) left_parent_record_id = [] if left_parent_record is None else [left_parent_record._id] right_parent_record_id = [] if right_parent_record is None else [right_parent_record._id] new_dr = DataRecord( schema(**data_item), source_indices=left_parent_source_indices + right_parent_source_indices, parent_ids=left_parent_record_id + right_parent_record_id, cardinality_idx=cardinality_idx, ) return new_dr @staticmethod def to_df(records: list[DataRecord], project_cols: list[str] | None = None) -> pd.DataFrame: if len(records) == 0: return pd.DataFrame() fields = records[0].get_field_names() if project_cols is not None and len(project_cols) > 0: fields = [field for field in fields if field in project_cols] # convert Context --> str for record in records: for k in fields: if isinstance(record[k], context.Context): record[k] = record[k].description return pd.DataFrame([ {k: record[k] for k in fields} for record in records ]) def to_json_str(self, include_bytes: bool = True, bytes_to_str: bool = False, project_cols: list[str] | None = None, sorted: bool = False): """Return a JSON representation of this DataRecord""" record_dict = self.to_dict(include_bytes, bytes_to_str, project_cols, sorted) return json.dumps(record_dict, indent=2) def to_dict(self, include_bytes: bool = True, bytes_to_str: bool = False, project_cols: list[str] | None = None, _sorted: bool = False, mask_filepaths: bool = False): """Return a dictionary representation of this DataRecord""" # TODO(chjun): In case of numpy types, the json.dumps will fail. Convert to native types. # Better ways to handle this. field_values = { k: v.description if isinstance(v, context.Context) else v for k, v in self._data_item.model_dump().items() } dct = pd.Series(field_values).to_dict() if project_cols is not None and len(project_cols) > 0: project_field_names = set(field.split(".")[-1] for field in project_cols) dct = {k: v for k, v in dct.items() if k in project_field_names} if not include_bytes: bytes_field_types = [bytes, list[bytes], bytes | None, list[bytes] | None, bytes | Any, list[bytes] | Any] bytes_field_types += AUDIO_FIELD_TYPES + IMAGE_FIELD_TYPES for k in dct: field_type = self.get_field_type(k) if field_type.annotation in bytes_field_types: dct[k] = "" if bytes_to_str: for k, v in dct.items(): if isinstance(v, bytes): dct[k] = v.decode("utf-8") elif isinstance(v, list) and len(v) > 0 and any([isinstance(elt, bytes) for elt in v]): dct[k] = [elt.decode("utf-8") if isinstance(elt, bytes) else elt for elt in v] if _sorted: dct = dict(sorted(dct.items())) if mask_filepaths: for k in dct: field_type = self.get_field_type(k) if field_type.annotation in [AudioBase64, AudioFilepath, ImageBase64, ImageFilepath, ImageURL]: dct[k] = "" return deepcopy(dct) class DataRecordSet: """ A DataRecordSet contains a list of DataRecords that share the same schema, same parent(s), and same source(s). We explicitly check that this is True. The record_op_stats could be empty if the DataRecordSet is not from executing an operator. """ def __init__( self, data_records: list[DataRecord], record_op_stats: list[RecordOpStats], field_to_score_fn: dict[str, str | callable] | None = None, input: int | DataRecord | list[DataRecord] | tuple[list[DataRecord]] | None = None, ): # set data_records, parent_ids, and source_indices; note that it is possible for # data_records to be an empty list in the event of a failed convert self.data_records = data_records self.parent_ids = data_records[0]._parent_ids if len(data_records) > 0 else None self.source_indices = data_records[0]._source_indices if len(data_records) > 0 else None self.schema = data_records[0].schema if len(data_records) > 0 else None # the input to the operator which produced the data_records; type is tuple[DataRecord] | tuple[int] # - for scan operators, input is a singleton tuple[int] which wraps the source_idx, e.g.: (source_idx,) # - for join operators, input is a tuple with one entry for the left input DataRecord and one entry for the right input DataRecord # - for aggregate operators, input is a tuple with all the input DataRecords to the aggregation # - for all other operaotrs, input is a singleton tuple[DataRecord] which wraps the single input self.input = input # set statistics for generating these records self.record_op_stats = record_op_stats # assign field_to_score_fn if provided self.field_to_score_fn = {} if field_to_score_fn is None else field_to_score_fn def get_total_cost(self) -> float: return sum([record_op_stats.cost_per_record for record_op_stats in self.record_op_stats]) def get_field_to_score_fn(self) -> dict[str, str | callable]: return self.field_to_score_fn def __getitem__(self, slice) -> DataRecord | list[DataRecord]: return self.data_records[slice] def __len__(self) -> int: return len(self.data_records) def __iter__(self) -> Generator[DataRecord]: yield from self.data_records class DataRecordCollection: """ A DataRecordCollection contains a list of DataRecords. This is a wrapper class for list[DataRecord] to support more advanced features for output of execute(). The difference between DataRecordSet and DataRecordCollection Goal: DataRecordSet is a set of DataRecords that share the same schema, same parents, and same sources. DataRecordCollection is a general wrapper for list[DataRecord]. Usage: DataRecordSet is used for the output of executing an operator. DataRecordCollection is used for the output of executing a query, we definitely could extend it to support more advanced features for output of execute(). """ def __init__(self, data_records: list[DataRecord], execution_stats: ExecutionStats | None = None, plan_stats: PlanStats | None = None): self.data_records = data_records self.execution_stats = execution_stats self.plan_stats = plan_stats self.executed_plans = self._get_executed_plans() def __iter__(self) -> Generator[DataRecord]: """Allow iterating directly over the data records""" yield from self.data_records def __len__(self): """Return the number of records in the collection""" return len(self.data_records) def to_df(self, cols: list[str] | None = None): return DataRecord.to_df(self.data_records, cols) def _get_executed_plans(self): if self.plan_stats is not None: return [self.plan_stats.plan_str] elif self.execution_stats is not None: return list(self.execution_stats.plan_strs.values()) else: return None ================================================ FILE: src/palimpzest/core/lib/__init__.py ================================================ ================================================ FILE: src/palimpzest/core/lib/schemas.py ================================================ from __future__ import annotations import sys from typing import Any, TypeAliasType import pandas as pd from pydantic import BaseModel, Field, create_model from pydantic.fields import FieldInfo from palimpzest.utils.hash_helpers import hash_for_serialized_dict # DEFINITIONS PANDAS_DTYPE_TO_PYDANTIC = { "object": str, "bool": bool, "int64": int, "float64": float, } # IMAGE TYPES ImageFilepath = TypeAliasType('ImageFilepath', str) ImageBase64 = TypeAliasType('ImageBase64', str) ImageURL = TypeAliasType('ImageURL', str) # AUDIO TYPES AudioFilepath = TypeAliasType('AudioFilepath', str) AudioBase64 = TypeAliasType('AudioBase64', str) IMAGE_LIST_FIELD_TYPES = [ list[ImageBase64], list[ImageFilepath], list[ImageURL], list[ImageBase64] | None, list[ImageFilepath] | None, list[ImageURL] | None, list[ImageBase64] | Any, list[ImageFilepath] | Any, list[ImageURL] | Any, ] IMAGE_FIELD_TYPES = IMAGE_LIST_FIELD_TYPES + [ ImageBase64, ImageFilepath, ImageURL, ImageBase64 | None, ImageFilepath | None, ImageURL | None, ImageBase64 | Any, ImageFilepath | Any, ImageURL | Any, ] AUDIO_LIST_FIELD_TYPES = [ list[AudioBase64], list[AudioFilepath], list[AudioBase64] | None, list[AudioFilepath] | None, list[AudioBase64] | Any, list[AudioFilepath] | Any, ] AUDIO_FIELD_TYPES = AUDIO_LIST_FIELD_TYPES + [ AudioBase64, AudioFilepath, AudioBase64 | None, AudioFilepath | None, AudioBase64 | Any, AudioFilepath | Any, ] def get_schema_field_names(schema: type[BaseModel], id: str | None = None) -> list[str]: """Return the field names of a Pydantic model.""" return list(schema.model_fields) if id is None else [f"{schema.__name__}.{id}.{field_name}" for field_name in schema.model_fields] def _create_pickleable_model(fields: dict[str, tuple[type, FieldInfo]]) -> type[BaseModel]: """Create a Pydantic model that can be pickled.""" # create unique name for the unioned model new_schema_name = f"Schema{sorted(fields.keys())}" new_schema_id = hash_for_serialized_dict({ field_name: {"annotation": str(annotation), "default": str(field.default), "description": field.description} for field_name, (annotation, field) in fields.items() }) # if this class already exists, get it from the module and return module = sys.modules[__name__] if hasattr(module, new_schema_id): return getattr(module, new_schema_id) # create the class dynamically new_model = create_model(new_schema_name, **fields) # register it in the module's namespace so pickle can find it module = sys.modules[__name__] setattr(module, new_schema_id, new_model) new_model.__module__ = module.__name__ return new_model def relax_schema(model: type[BaseModel]) -> type[BaseModel]: """Updates the type annotation for every field in the BaseModel to include typing.Any""" fields = {} for field_name, field in model.model_fields.items(): fields[field_name] = (field.annotation | Any, field) return _create_pickleable_model(fields) def project(model: type[BaseModel], project_fields: list[str]) -> type[BaseModel]: """Project a Pydantic model to only the specified columns.""" # make sure projection column names are shortened project_fields = [field_name.split(".")[-1] for field_name in project_fields] # build up the fields for the new schema fields = {} for field_name, field in model.model_fields.items(): if field_name in project_fields: fields[field_name] = (field.annotation, field) # create and return the new schema return _create_pickleable_model(fields) def create_schema_from_fields(fields: list[dict]) -> type[BaseModel]: """Create a Pydantic model from a list of fields.""" fields_ = {} for field in fields: assert "name" in field, "fields must contain a 'name' key" assert "type" in field, "fields must contain a 'type' key" assert "desc" in field or "description" in field, "fields must contain a 'description' key" # for backwards compatability, rename "desc" to "description" if "desc" in field: field["description"] = field.pop("desc") field_name = field["name"] field_type = field["type"] fields_[field_name] = (field_type, Field(**{k: v for k, v in field.items() if k not in ["name", "type"]})) return _create_pickleable_model(fields_) def create_schema_from_df(df: pd.DataFrame) -> type[BaseModel]: """Create a Pydantic model from a Pandas DataFrame.""" fields = {} for column, dtype in zip(df.columns, df.dtypes): column = f"column_{column}" if isinstance(column, int) else column field_desc = f"The {column} column from an input DataFrame" annotation = PANDAS_DTYPE_TO_PYDANTIC.get(str(dtype), Any) fields[column] = (annotation, Field(description=field_desc)) # create and return the new schema return _create_pickleable_model(fields) def union_schemas(models: list[type[BaseModel]], join: bool = False, on: list[str] | None = None) -> type[BaseModel]: """Union multiple Pydantic models into a single model.""" # convert on to empty list if None if on is None: on = [] # build up the fields for the new schema fields = {} for model in models: for field_name, field in model.model_fields.items(): # for non-join unions, make sure duplicate fields have the same type if not join and field_name in fields: assert fields[field_name][0] == field.annotation, f"Field {field_name} has different types in different models" # for joins with "on" specified, no need to rename fields in "on" elif join and field_name in on and field_name in fields: continue # otherwise, rename duplicate fields by appending _right elif join and field_name in fields: while field_name in fields: field_name = f"{field_name}_right" # add the field to the new schema fields[field_name] = (field.annotation, field) # create and return the new schema return _create_pickleable_model(fields) ################################################################################### # "Core" useful Schemas. These are Schemas that almost everyone will need. # File, TextFile, Image, PDF, etc. ################################################################################### # First-level Schema's class DefaultSchema(BaseModel): """Store context data.""" value: Any = Field(description="The value of the input data") class Download(BaseModel): """A download is a URL and the contents of the download.""" url: str = Field(description="The URL of the download") content: bytes = Field(description="The contents of the download") timestamp: str = Field(description="The timestamp of the download") class File(BaseModel): """ A File is defined by two Fields: - the filename (string) - the contents of the file (bytes) """ filename: str = Field(description="The UNIX-style name of the file") contents: bytes = Field(description="The contents of the file") class TextFile(BaseModel): """A text file is a File that contains only text. No binary data.""" filename: str = Field(description="The UNIX-style name of the file") contents: str = Field(description="The contents of the file") class Average(BaseModel): average: float = Field(description="The average value of items in the dataset") class Count(BaseModel): count: int = Field(description="The count of items in the dataset") class Sum(BaseModel): sum: int = Field(description="The summation of items in the dataset") class Min(BaseModel): min: int | float = Field(description="The minimum value of some items in the dataset") class Max(BaseModel): max: int | float = Field(description="The maximum value of some items in the dataset") class OperatorDerivedSchema(BaseModel): """Schema defined by an operator, e.g., a join or a group by""" class Table(BaseModel): """A Table is an object composed of a header and rows.""" filename: str = Field(description="The name of the file the table was extracted from") name: str = Field(description="The name of the table") header: list[str] = Field(description="The header of the table") rows: list[list] = Field(description="The rows of the table") class URL(BaseModel): """A URL is a string that represents a web address.""" url: str = Field(description="A URL") class WebPage(BaseModel): """A web page is a URL and the contents of the page.""" text: str = Field(description="The text contents of the web page") html: str = Field(description="The html contents of the web page") timestamp: str = Field(description="The timestamp of the download") filename: str = Field(description="The name of the file the web page was downloaded from") # Second-level Schemas class ImageFile(File): """A file that contains an image.""" contents: ImageBase64 = Field(description="The contents of the image encoded as a base64 string") class AudioFile(File): """A file that contains audio.""" contents: AudioBase64 = Field(description="The contents of an audio recording encoded as a base64 string") class PDFFile(File): """A PDF file is a File that is a PDF. It has specialized fields, font information, etc.""" # This class is currently very impoverished. It needs a lot more fields before it can correctly represent a PDF. text_contents: str = Field(description="The text-only contents of the PDF") class XLSFile(File): """An XLS file is a File that contains one or more Excel spreadsheets.""" number_sheets: int = Field(description="The number of sheets in the Excel file") sheet_names: list[str] = Field(description="The names of the sheets in the Excel file") # Third-level Schemas class EquationImage(ImageFile): """An image that contains a mathematical equation.""" equation_text: str = Field(description="The text representation of the equation in the image") class PlotImage(ImageFile): """An image that contains a plot, such as a graph or chart.""" plot_description: str = Field(description="A description of the plot") ================================================ FILE: src/palimpzest/core/models.py ================================================ from __future__ import annotations import json import time from abc import abstractmethod from typing import Any from pydantic import BaseModel, Field class GenerationStats(BaseModel): """ Model for storing statistics about the execution of an operator on a single record. """ model_name: str | None = None # The raw answer as output from the generator (a list of strings, possibly of len 1) # raw_answers: Optional[List[str]] = field(default_factory=list) # the total number of input text tokens processed by this operator; None if this operation did not use any LLM # typed as a float because GenerationStats may be amortized (i.e. divided) acorss a number of output records input_text_tokens: float = 0.0 # the total number of input audio tokens processed by this operation. input_audio_tokens: float = 0.0 # the total number of input image tokens processed by this operation. input_image_tokens: float = 0.0 # the total number of cache read tokens processed by this operation (charged at a discount, typically 0.1x input rate) cache_read_tokens: float = 0.0 # the total number of tokens written to the cache in this operation (Anthropic only) (charged at creation rate, typically 1.25x input rate) cache_creation_tokens: float = 0.0 # the number of output text tokens generated by the model output_text_tokens: float = 0.0 # the total number of input tokens processed by embedding models embedding_input_tokens: float = 0.0 # the total cost of processing the input and output tokens; None if this operation did not use an LLM # TODO: future PR: cost_per_record --> total_cost cost_per_record: float = 0.0 # (if applicable) the time (in seconds) spent executing a call to an LLM llm_call_duration_secs: float = 0.0 # (if applicable) the time (in seconds) spent executing a call to a function fn_call_duration_secs: float = 0.0 # (if applicable) the total number of LLM calls made by this operator total_llm_calls: float = 0.0 # (if applicable) the total number of embedding LLM calls made by this operator total_embedding_llm_calls: float = 0.0 def __iadd__(self, other: GenerationStats) -> GenerationStats: for field in type(self).model_fields: if field == "model_name": continue setattr(self, field, getattr(self, field) + getattr(other, field)) return self def __add__(self, other: GenerationStats) -> GenerationStats: dct = { field: getattr(self, field) + getattr(other, field) for field in type(self).model_fields if field != "model_name" } dct["model_name"] = self.model_name return GenerationStats(**dct) # Do the same as iadd and add but with division operator def __itruediv__(self, quotient: float) -> GenerationStats: if quotient == 0: raise ZeroDivisionError("Cannot divide by zero") if isinstance(quotient, int): quotient = float(quotient) for field in type(self).model_fields: if field == "model_name": continue setattr(self, field, getattr(self, field) / quotient) return self def __truediv__(self, quotient: float) -> GenerationStats: if quotient == 0: raise ZeroDivisionError("Cannot divide by zero") if isinstance(quotient, int): quotient = float(quotient) dct = { field: getattr(self, field) / quotient for field in type(self).model_fields if field != "model_name" } dct["model_name"] = self.model_name return GenerationStats(**dct) def __radd__(self, other: int) -> GenerationStats: assert not isinstance(other, GenerationStats), "This should not be called with a GenerationStats object" return self # NOTE: this is added temporarily to help track cost of compute agent writing PZ code; # once we find a long-term solution for tracking that cost, we can remove this def to_json(self, filepath: str | None = None) -> dict | None: if filepath is None: return self.model_dump(mode="json") with open(filepath, "w") as f: json.dump(self.model_dump(mode="json"), f) class RecordOpStats(BaseModel): """ Model for storing statistics about the execution of an operator on a single record. """ ##### REQUIRED FIELDS ##### # record id; an identifier for this record record_id: str | int # identifier for the parent(s) of this record record_parent_ids: list[str | int] | None # idenifier for the source indices of this record record_source_indices: list[str | int] # a dictionary with the record state after being processed by the operator record_state: dict[str, Any] # operation id; an identifier for this operation's physical op id full_op_id: str # logical operation id; the logical op id for this physical op logical_op_id: str # operation name op_name: str # the time spent by the data record just in this operation time_per_record: float # the cost (in dollars) to generate this record at this operation cost_per_record: float ##### NOT-OPTIONAL, BUT FILLED BY EXECUTION CLASS AFTER CONSTRUCTOR CALL ##### # the ID(s) of the physical operation(s) which produced the input record(s) for this record at this operation source_unique_full_op_ids: list[str] | None = None # the ID(s) of the logical operation(s) which produced the input record(s) for this record at this operation source_unique_logical_op_ids: list[str] | None = None # the ID of the physical plan which produced this record at this operation plan_id: str = "" ##### OPTIONAL, BUT FILLED BY COST MODEL AFTER SAMPLE DATA EXECUTION ##### quality: float | None = None ##### OPTIONAL FIELDS (I.E. ONLY MANDATORY FOR CERTAIN OPERATORS) ##### # (if applicable) the name of the model used to generate the output for this record model_name: str | None = None # (if applicable) the mapping from field-name to generated output for this record answer: dict[str, Any] | None = None # (if applicable) the mapping from field-name to generated output for this record # raw_answers: Optional[List[str, Any]] = field(default_factory=list) # (if applicable) the list of input fields for the generation for this record input_fields: list[str] | None = None # (if applicable) the list of generated fields for this record generated_fields: list[str] | None = None # the number of input text tokens processed by this operation # typed as a float because GenerationStats may be amortized (i.e. divided) across a number of output records input_text_tokens: float = 0.0 # the number of input audio tokens processed by this operation input_audio_tokens: float = 0.0 # the number of input image tokens processed by this operation input_image_tokens: float = 0.0 # the number of cache read tokens processed by this operation cache_read_tokens: float = 0.0 # the number of tokens written to cache by this operation cache_creation_tokens: float = 0.0 # the number of output text tokens generated by this operation output_text_tokens: float = 0.0 # the number of input tokens processed by embedding models embedding_input_tokens: float = 0.0 # (if applicable) the filter text (or a string representation of the filter function) applied to this record filter_str: str | None = None # (if applicable) the join condition applied to this record join_condition: str | None = None # the True/False result of whether this record was output by the operator or not # (can only be False if the operator is a Filter or Join) passed_operator: bool = True # (if applicable) the time (in seconds) spent executing a call to an LLM llm_call_duration_secs: float = 0.0 # (if applicable) the time (in seconds) spent executing a UDF or calling an external api fn_call_duration_secs: float = 0.0 # (if applicable) the total number of LLM calls made by this operator total_llm_calls: float = 0.0 # (if applicable) the total number of embedding LLM calls made by this operator total_embedding_llm_calls: float = 0.0 # (if applicable) a boolean indicating whether this is the statistics captured from a failed convert operation failed_convert: bool | None = None # an OPTIONAL dictionary with more detailed information about this operation; op_details: dict[str, Any] = Field(default_factory=dict) class OperatorStats(BaseModel): """ Model for storing statistics captured within a given operator. """ # the full ID of the physical operation in which these stats were collected full_op_id: str # the name of the physical operation in which these stats were collected op_name: str # the total time spent in this operation total_op_time: float = 0.0 # the total cost of this operation total_op_cost: float = 0.0 # the number of input text tokens processed by this operation input_text_tokens: float = 0.0 # the number of input audio tokens processed by this operation input_audio_tokens: float = 0.0 # the number of input image tokens processed by this operation input_image_tokens: float = 0.0 # the number of cache read tokens processed by this operation cache_read_tokens: float = 0.0 # the number of tokens written to cache by this operation cache_creation_tokens: float = 0.0 # the number of output text tokens generated by this operation output_text_tokens: float = 0.0 # the number of input tokens processed by embedding models embedding_input_tokens: float = 0.0 # a list of RecordOpStats processed by the operation record_op_stats_lst: list[RecordOpStats] = Field(default_factory=list) # the unique full ID(s) of the physical operator(s) which precede this one (used by PlanStats) source_unique_full_op_ids: list[str] | None = None # the unique full ID(s) of the logical operator(s) which precede this one (used by SentinelPlanStats) source_unique_logical_op_ids: list[str] | None = None # the ID of the physical plan which this operator is part of plan_id: str = "" # an OPTIONAL dictionary with more detailed information about this operation; op_details: dict[str, Any] = Field(default_factory=dict) def __iadd__(self, stats: OperatorStats | RecordOpStats) -> OperatorStats: """ Sum the given stats to this operator's stats. The given stats can be either: 1. an OperatorStats object 2. a RecordOpStats object NOTE: in case (1.) we assume the execution layer guarantees that `stats` is generated by the same operator in the same plan. Thus, we assume the full_op_ids, op_name, source_op_id, etc. do not need to be updated. """ if isinstance(stats, OperatorStats): self.total_op_time += stats.total_op_time self.total_op_cost += stats.total_op_cost self.input_text_tokens += stats.input_text_tokens self.input_audio_tokens += stats.input_audio_tokens self.input_image_tokens += stats.input_image_tokens self.cache_read_tokens += stats.cache_read_tokens self.cache_creation_tokens += stats.cache_creation_tokens self.output_text_tokens += stats.output_text_tokens self.embedding_input_tokens += stats.embedding_input_tokens self.record_op_stats_lst.extend(stats.record_op_stats_lst) elif isinstance(stats, RecordOpStats): stats.source_unique_full_op_ids = self.source_unique_full_op_ids stats.plan_id = self.plan_id self.record_op_stats_lst.append(stats) self.total_op_time += stats.time_per_record self.total_op_cost += stats.cost_per_record self.input_text_tokens += stats.input_text_tokens self.input_audio_tokens += stats.input_audio_tokens self.input_image_tokens += stats.input_image_tokens self.cache_read_tokens += stats.cache_read_tokens self.cache_creation_tokens += stats.cache_creation_tokens self.output_text_tokens += stats.output_text_tokens self.embedding_input_tokens += stats.embedding_input_tokens else: raise TypeError(f"Cannot add {type(stats)} to OperatorStats") return self class BasePlanStats(BaseModel): """ Model for storing statistics captured for an entire plan. This class is subclassed for tracking: - PlanStats: the statistics for execution of a PhysicalPlan - SentinelPlanStats: the statistics for execution of a SentinelPlan The key difference between the two subclasses is that the `operator_stats` field in the PlanStats maps from the physical operator ids to their corresponding OperatorStats objects. The `operator_stats` field in the SentinelPlanStats maps from a logical operator id to another dictionary which maps from the physical operator ids to their corresponding OperatorStats objects. """ # id for identifying the physical plan plan_id: str # string representation of the physical plan plan_str: str | None = None # dictionary whose values are OperatorStats objects; # PlanStats maps {full_op_id -> OperatorStats} # SentinelPlanStats maps {logical_op_id -> {full_op_id -> OperatorStats}} operator_stats: dict[str, OperatorStats | dict[str, OperatorStats]] = Field(default_factory=dict) # dictionary whose values are GenerationStats objects for validation; # only used by SentinelPlanStats validation_gen_stats: dict[str, GenerationStats] = Field(default_factory=dict) # total runtime for the plan measured from the start to the end of PhysicalPlan.execute() total_plan_time: float = 0.0 # total cost for plan total_plan_cost: float = 0.0 # input text tokens processed by this plan input_text_tokens: float = 0.0 # input audio tokens processed by this plan input_audio_tokens: float = 0.0 # input image tokens processed by this plan input_image_tokens: float = 0.0 # cache read tokens processed by this plan cache_read_tokens: float = 0.0 # tokens written to cache by this plan cache_creation_tokens: float = 0.0 # output text tokens generated by this plan output_text_tokens: float = 0.0 # embedding input tokens processed by this plan embedding_input_tokens: float = 0.0 # start time for the plan execution; should be set by calling PlanStats.start() start_time: float | None = None def start(self) -> None: """Start the timer for this plan execution.""" self.start_time = time.time() def finish(self) -> None: """Finish the timer for this plan execution.""" if self.start_time is None: raise RuntimeError("PlanStats.start() must be called before PlanStats.finish()") self.total_plan_time = time.time() - self.start_time self.total_plan_cost = self.sum_op_stats_field("total_op_cost") + self.sum_validation_stats_field("cost_per_record") self.input_text_tokens = self.sum_op_stats_field("input_text_tokens") + self.sum_validation_stats_field("input_text_tokens") self.input_audio_tokens = self.sum_op_stats_field("input_audio_tokens") + self.sum_validation_stats_field("input_audio_tokens") self.input_image_tokens = self.sum_op_stats_field("input_image_tokens") + self.sum_validation_stats_field("input_image_tokens") self.cache_read_tokens = self.sum_op_stats_field("cache_read_tokens") + self.sum_validation_stats_field("cache_read_tokens") self.cache_creation_tokens = self.sum_op_stats_field("cache_creation_tokens") + self.sum_validation_stats_field("cache_creation_tokens") self.output_text_tokens = self.sum_op_stats_field("output_text_tokens") + self.sum_validation_stats_field("output_text_tokens") self.embedding_input_tokens = self.sum_op_stats_field("embedding_input_tokens") + self.sum_validation_stats_field("embedding_input_tokens") @staticmethod @abstractmethod def from_plan(plan) -> BasePlanStats: """ Initialize this PlanStats object from a PhysicalPlan or SentinelPlan object. """ pass @abstractmethod def sum_op_stats_field(self, field_name: str) -> float | int: """Sum a given field across all operator stats in this plan.""" pass def sum_validation_stats_field(self, field_name: str) -> float | int: """Sum a given field across all validation generation stats in this plan.""" return sum([getattr(gen_stats, field_name) for _, gen_stats in self.validation_gen_stats.items()]) @abstractmethod def add_record_op_stats(self, unique_full_op_id: str, record_op_stats: RecordOpStats | list[RecordOpStats]) -> None: """ Add the given RecordOpStats to this plan's operator stats for the given operator id. """ pass @abstractmethod def __iadd__(self, plan_stats: BasePlanStats) -> None: """ Add the given PlanStats to this plan's operator stats. """ pass @abstractmethod def __str__(self) -> str: """ Return a string representation of this plan's statistics. """ pass def get_total_cost_so_far(self) -> float: """ Get the total cost incurred so far in this plan execution. """ return self.sum_op_stats_field("total_op_cost") + self.sum_validation_stats_field("cost_per_record") class PlanStats(BasePlanStats): """ Subclass of BasePlanStats which captures statistics from the execution of a single PhysicalPlan. """ @staticmethod def from_plan(plan) -> PlanStats: """ Initialize this PlanStats object from a PhysicalPlan object. """ # TODO?: have PhysicalPlan return PlanStats object operator_stats = {} for topo_idx, op in enumerate(plan): unique_full_op_id = f"{topo_idx}-{op.get_full_op_id()}" operator_stats[unique_full_op_id] = OperatorStats( full_op_id=op.get_full_op_id(), op_name=op.op_name(), source_unique_full_op_ids=plan.get_source_unique_full_op_ids(topo_idx, op), plan_id=plan.plan_id, op_details={k: str(v) for k, v in op.get_id_params().items()}, ) return PlanStats(plan_id=plan.plan_id, plan_str=str(plan), operator_stats=operator_stats) def sum_op_stats_field(self, field_name: str) -> float | int: """Sum a given field across all operator stats in this plan.""" return sum([getattr(op_stats, field_name) for _, op_stats in self.operator_stats.items()]) def add_record_op_stats(self, unique_full_op_id: str, record_op_stats: RecordOpStats | list[RecordOpStats]) -> None: """ Add the given RecordOpStats to this plan's operator stats for the given operator id. """ # normalize input type to be list[RecordOpStats] record_op_stats_lst = record_op_stats if isinstance(record_op_stats, list) else [record_op_stats] # update operator stats for record_op_stats in record_op_stats_lst: if unique_full_op_id in self.operator_stats: self.operator_stats[unique_full_op_id] += record_op_stats else: raise ValueError(f"RecordOpStats with unique_full_op_id {unique_full_op_id} not found in PlanStats") def __iadd__(self, plan_stats: PlanStats) -> None: """ NOTE: we assume the execution layer guarantees: 1. these plan_stats belong to the same plan 2. these plan_stats come from sequential (non-overlapping) executions of the same plan The latter criteria implies it is okay for this method to sum the plan (and operator) runtimes. """ self.total_plan_time += plan_stats.total_plan_time self.total_plan_cost += plan_stats.total_plan_cost self.input_text_tokens += plan_stats.input_text_tokens self.input_audio_tokens += plan_stats.input_audio_tokens self.input_image_tokens += plan_stats.input_image_tokens self.cache_read_tokens += plan_stats.cache_read_tokens self.cache_creation_tokens += plan_stats.cache_creation_tokens self.output_text_tokens += plan_stats.output_text_tokens self.embedding_input_tokens += plan_stats.embedding_input_tokens for unique_full_op_id, op_stats in plan_stats.operator_stats.items(): if unique_full_op_id in self.operator_stats: self.operator_stats[unique_full_op_id] += op_stats else: self.operator_stats[unique_full_op_id] = op_stats def __str__(self) -> str: stats = f"total_plan_time={self.total_plan_time} \n" stats += f"total_plan_cost={self.total_plan_cost} \n" stats += f"input_text_tokens={self.input_text_tokens} \n" stats += f"input_audio_tokens={self.input_audio_tokens} \n" stats += f"input_image_tokens={self.input_image_tokens} \n" stats += f"cache_read_tokens={self.cache_read_tokens} \n" stats += f"cache_creation_tokens={self.cache_creation_tokens} \n" stats += f"output_text_tokens={self.output_text_tokens} \n" stats += f"embedding_input_tokens={self.embedding_input_tokens} \n" for idx, op_stats in enumerate(self.operator_stats.values()): stats += f"{idx}. {op_stats.op_name} time={op_stats.total_op_time} cost={op_stats.total_op_cost} \n" return stats class SentinelPlanStats(BasePlanStats): """ Subclass of BasePlanStats which captures statistics from the execution of a single SentinelPlan. """ @staticmethod def from_plan(plan) -> SentinelPlanStats: """ Initialize this PlanStats object from a Sentinel object. """ operator_stats = {} for topo_idx, (logical_op_id, op_set) in enumerate(plan): unique_logical_op_id = f"{topo_idx}-{logical_op_id}" operator_stats[unique_logical_op_id] = {} for physical_op in op_set: full_op_id = physical_op.get_full_op_id() operator_stats[unique_logical_op_id][full_op_id] = OperatorStats( full_op_id=full_op_id, op_name=physical_op.op_name(), source_unique_logical_op_ids=plan.get_source_unique_logical_op_ids(unique_logical_op_id), plan_id=plan.plan_id, op_details={k: str(v) for k, v in physical_op.get_id_params().items()}, ) return SentinelPlanStats(plan_id=plan.plan_id, plan_str=str(plan), operator_stats=operator_stats) def sum_op_stats_field(self, field_name: str) -> float | int: """Sum a given field across all operator stats in this plan.""" return sum(sum([getattr(op_stats, field_name) for _, op_stats in phys_op_stats.items()]) for _, phys_op_stats in self.operator_stats.items()) def add_record_op_stats(self, unique_logical_op_id: str, record_op_stats: RecordOpStats | list[RecordOpStats]) -> None: """ Add the given RecordOpStats to this plan's operator stats for the given operator set id. """ # normalize input type to be list[RecordOpStats] record_op_stats_lst = record_op_stats if isinstance(record_op_stats, list) else [record_op_stats] # update operator stats for record_op_stats in record_op_stats_lst: full_op_id = record_op_stats.full_op_id if unique_logical_op_id in self.operator_stats: if full_op_id in self.operator_stats[unique_logical_op_id]: self.operator_stats[unique_logical_op_id][full_op_id] += record_op_stats else: raise ValueError(f"RecordOpStats with full_op_id {full_op_id} not found in SentinelPlanStats") else: raise ValueError(f"RecordOpStats with unique_logical_op_id {unique_logical_op_id} not found in SentinelPlanStats") def add_validation_gen_stats(self, unique_logical_op_id: str, gen_stats: GenerationStats) -> None: """ Add the given GenerationStats to this plan's validation generation stats for the given logical operator id. """ if unique_logical_op_id in self.validation_gen_stats: self.validation_gen_stats[unique_logical_op_id] += gen_stats else: self.validation_gen_stats[unique_logical_op_id] = gen_stats def __iadd__(self, plan_stats: SentinelPlanStats) -> None: """ NOTE: we assume the execution layer guarantees: 1. these plan_stats belong to the same plan 2. these plan_stats come from sequential (non-overlapping) executions of the same plan The latter criteria implies it is okay for this method to sum the plan (and operator) runtimes. """ self.total_plan_time += plan_stats.total_plan_time self.total_plan_cost += plan_stats.total_plan_cost self.input_text_tokens += plan_stats.input_text_tokens self.input_audio_tokens += plan_stats.input_audio_tokens self.input_image_tokens += plan_stats.input_image_tokens self.cache_read_tokens += plan_stats.cache_read_tokens self.cache_creation_tokens += plan_stats.cache_creation_tokens self.output_text_tokens += plan_stats.output_text_tokens self.embedding_input_tokens += plan_stats.embedding_input_tokens for unique_logical_op_id, physical_op_stats in plan_stats.operator_stats.items(): for full_op_id, op_stats in physical_op_stats.items(): if unique_logical_op_id in self.operator_stats: if full_op_id in self.operator_stats[unique_logical_op_id]: self.operator_stats[unique_logical_op_id][full_op_id] += op_stats else: self.operator_stats[unique_logical_op_id][full_op_id] = op_stats else: self.operator_stats[unique_logical_op_id] = physical_op_stats for unique_logical_op_id, gen_stats in plan_stats.validation_gen_stats.items(): if unique_logical_op_id in self.validation_gen_stats: self.validation_gen_stats[unique_logical_op_id] += gen_stats else: self.validation_gen_stats[unique_logical_op_id] = gen_stats def __str__(self) -> str: stats = f"total_plan_time={self.total_plan_time} \n" stats += f"total_plan_cost={self.total_plan_cost} \n" stats += f"input_text_tokens={self.input_text_tokens} \n" stats += f"input_audio_tokens={self.input_audio_tokens} \n" stats += f"input_image_tokens={self.input_image_tokens} \n" stats += f"cache_read_tokens={self.cache_read_tokens} \n" stats += f"cache_creation_tokens={self.cache_creation_tokens} \n" stats += f"output_text_tokens={self.output_text_tokens} \n" stats += f"embedding_input_tokens={self.embedding_input_tokens} \n" for outer_idx, physical_op_stats in enumerate(self.operator_stats.values()): total_time = sum([op_stats.total_op_time for op_stats in physical_op_stats.values()]) total_cost = sum([op_stats.total_op_cost for op_stats in physical_op_stats.values()]) stats += f"{outer_idx}. total_time={total_time} total_cost={total_cost} \n" for inner_idx, op_stats in enumerate(physical_op_stats.values()): stats += f" {outer_idx}.{inner_idx}. {op_stats.op_name} time={op_stats.total_op_time} cost={op_stats.total_op_cost} \n" return stats class ExecutionStats(BaseModel): """ Model for storing statistics captured for the entire execution of a workload. """ # string for identifying this workload execution execution_id: str | None = None # dictionary of SentinelPlanStats objects (one for each sentinel plan run during execution) sentinel_plan_stats: dict[str, SentinelPlanStats] = Field(default_factory=dict) # dictionary of PlanStats objects (one for each plan run during execution) plan_stats: dict[str, PlanStats] = Field(default_factory=dict) # total time spent optimizing optimization_time: float = 0.0 # total cost of optimizing optimization_cost: float = 0.0 # total time spent executing the optimized plan plan_execution_time: float = 0.0 # total cost of executing the optimized plan plan_execution_cost: float = 0.0 # total runtime for the entire execution total_execution_time: float = 0.0 # total cost for the entire execution total_execution_cost: float = 0.0 # input text tokens processed input_text_tokens: float = 0.0 # input audio tokens processed input_audio_tokens: float = 0.0 # input image tokens processed input_image_tokens: float = 0.0 # cache read tokens processed cache_read_tokens: float = 0.0 # tokens written to cache cache_creation_tokens: float = 0.0 # output text tokens generated output_text_tokens: float = 0.0 # embedding input tokens processed embedding_input_tokens: float = 0.0 # dictionary of sentinel plan strings; useful for printing executed sentinel plans in demos sentinel_plan_strs: dict[str, str] = Field(default_factory=dict) # dictionary of plan strings; useful for printing executed plans in demos plan_strs: dict[str, str] = Field(default_factory=dict) # start time for the execution; should be set by calling ExecutionStats.start() start_time: float | None = None # end time for the optimization; optimization_end_time: float | None = None def start(self) -> None: """Start the timer for this execution.""" self.start_time = time.time() def finish_optimization(self) -> None: """Finish the timer for the optimization phase of this execution.""" if self.start_time is None: raise RuntimeError("ExecutionStats.start() must be called before ExecutionStats.finish_optimization()") # compute optimization time and cost self.optimization_end_time = time.time() self.optimization_time = self.optimization_end_time - self.start_time self.optimization_cost = self.sum_sentinel_plan_costs() # compute sentinel_plan_strs self.sentinel_plan_strs = {plan_id: plan_stats.plan_str for plan_id, plan_stats in self.sentinel_plan_stats.items()} def finish(self) -> None: """Finish the timer for this execution.""" if self.start_time is None: raise RuntimeError("ExecutionStats.start() must be called before ExecutionStats.finish()") # compute time for plan and total execution end_time = time.time() self.plan_execution_time = ( end_time - self.optimization_end_time if self.optimization_end_time is not None else end_time - self.start_time ) self.total_execution_time = end_time - self.start_time # compute the cost for plan and total execution self.plan_execution_cost = self.sum_plan_costs() self.total_execution_cost = self.optimization_cost + self.plan_execution_cost # compute the tokens for total execution self.input_text_tokens = self.sum_plan_stats_field("input_text_tokens") self.input_audio_tokens = self.sum_plan_stats_field("input_audio_tokens") self.input_image_tokens = self.sum_plan_stats_field("input_image_tokens") self.cache_read_tokens = self.sum_plan_stats_field("cache_read_tokens") self.cache_creation_tokens = self.sum_plan_stats_field("cache_creation_tokens") self.output_text_tokens = self.sum_plan_stats_field("output_text_tokens") self.embedding_input_tokens = self.sum_plan_stats_field("embedding_input_tokens") # compute plan_strs self.plan_strs = {plan_id: plan_stats.plan_str for plan_id, plan_stats in self.plan_stats.items()} def sum_plan_stats_field(self, field_name: str) -> float | int: """ Sum a given field across all PlanStats in this execution. """ sentinel_plan_field_sum = sum([plan_stats.sum_op_stats_field(field_name) + plan_stats.sum_validation_stats_field(field_name) for _, plan_stats in self.sentinel_plan_stats.items()]) plan_field_sum = sum([plan_stats.sum_op_stats_field(field_name) for _, plan_stats in self.plan_stats.items()]) return plan_field_sum + sentinel_plan_field_sum def sum_sentinel_plan_costs(self) -> float: """ Sum the costs of all SentinelPlans in this execution. """ return sum([ plan_stats.sum_op_stats_field("total_op_cost") + plan_stats.sum_validation_stats_field("cost_per_record") for _, plan_stats in self.sentinel_plan_stats.items() ]) def sum_plan_costs(self) -> float: """ Sum the costs of all PhysicalPlans in this execution. """ return sum([plan_stats.sum_op_stats_field("total_op_cost") for _, plan_stats in self.plan_stats.items()]) def add_plan_stats(self, plan_stats: PlanStats | SentinelPlanStats | list[PlanStats] | list[SentinelPlanStats]) -> None: """ Add the given PlanStats (or SentinelPlanStats) to this execution's plan stats. NOTE: we make the assumption that the same plan cannot be run more than once in parallel, i.e. each plan stats object for an individual plan comes from two different (sequential) periods in time. Thus, PlanStats objects can be summed. """ # normalize input type to be list[PlanStats] or list[SentinelPlanStats] if isinstance(plan_stats, (PlanStats, SentinelPlanStats)): plan_stats = [plan_stats] for plan_stats_obj in plan_stats: if isinstance(plan_stats_obj, PlanStats) and plan_stats_obj.plan_id not in self.plan_stats: self.plan_stats[plan_stats_obj.plan_id] = plan_stats_obj elif isinstance(plan_stats_obj, PlanStats): self.plan_stats[plan_stats_obj.plan_id] += plan_stats_obj elif isinstance(plan_stats_obj, SentinelPlanStats) and plan_stats_obj.plan_id not in self.sentinel_plan_stats: self.sentinel_plan_stats[plan_stats_obj.plan_id] = plan_stats_obj elif isinstance(plan_stats_obj, SentinelPlanStats): self.sentinel_plan_stats[plan_stats_obj.plan_id] += plan_stats_obj else: raise TypeError(f"Cannot add {type(plan_stats)} to ExecutionStats") def to_json(self, filepath: str | None = None) -> dict | None: if filepath is None: return self.model_dump(mode="json") with open(filepath, "w") as f: json.dump(self.model_dump(mode="json"), f) class OperatorCostEstimates(BaseModel): """ Model for storing estimates of key metrics of interest for each operator. """ # (estimated) number of records output by this operator cardinality: float # (estimated) avg. time spent in this operator per-record time_per_record: float # (estimated) dollars spent per-record by this operator cost_per_record: float # (estimated) quality of the output from this operator quality: float # lower bound on cardinality cardinality_lower_bound: float | None = None # upper bound on cardinality cardinality_upper_bound: float | None = None # lower bound on time_per_record time_per_record_lower_bound: float | None = None # upper bound on time_per_record time_per_record_upper_bound: float | None = None # lower bound on cost_per_record cost_per_record_lower_bound: float | None = None # upper bound on cost_per_record cost_per_record_upper_bound: float | None = None # lower bound on quality quality_lower_bound: float | None = None # upper bound on quality quality_upper_bound: float | None = None def __rmul__(self, multiplier: float) -> OperatorCostEstimates: """ Multiply all fields by a scalar. """ dct = {field_name: getattr(self, field_name) * multiplier for field_name in self.model_fields} return OperatorCostEstimates(**dct) def model_post_init(self, __context: Any) -> None: if self.cardinality_lower_bound is None and self.cardinality_upper_bound is None: self.cardinality_lower_bound = self.cardinality self.cardinality_upper_bound = self.cardinality if self.time_per_record_lower_bound is None and self.time_per_record_upper_bound is None: self.time_per_record_lower_bound = self.time_per_record self.time_per_record_upper_bound = self.time_per_record if self.cost_per_record_lower_bound is None and self.cost_per_record_upper_bound is None: self.cost_per_record_lower_bound = self.cost_per_record self.cost_per_record_upper_bound = self.cost_per_record if self.quality_lower_bound is None and self.quality_upper_bound is None: self.quality_lower_bound = self.quality self.quality_upper_bound = self.quality class PlanCost(BaseModel): """ Model for storing the (cost, time, quality) estimates of (sub)-plans and their upper and lower bounds. """ # the expression cost cost: float # the expression runtime time: float # the expression quality quality: float # operator-specific cost estimates op_estimates: OperatorCostEstimates | None = None # lower bound on the expression cost cost_lower_bound: float | None = None # upper bound on the expression cost cost_upper_bound: float | None = None # lower bound on the expression time time_lower_bound: float | None = None # upper bound on the expression time time_upper_bound: float | None = None # lower bound on the expression quality quality_lower_bound: float | None = None # upper bound on the expression quality quality_upper_bound: float | None = None def __hash__(self): return hash(f"{self.cost}-{self.time}-{self.quality}") def __eq__(self, other: Any) -> bool: if not isinstance(other, PlanCost): return False return ( self.cost == other.cost and self.time == other.time and self.quality == other.quality ) def model_post_init(self, __context: Any) -> None: if self.time_lower_bound is None and self.time_upper_bound is None: self.time_lower_bound = self.time self.time_upper_bound = self.time if self.cost_lower_bound is None and self.cost_upper_bound is None: self.cost_lower_bound = self.cost self.cost_upper_bound = self.cost if self.quality_lower_bound is None and self.quality_upper_bound is None: self.quality_lower_bound = self.quality self.quality_upper_bound = self.quality def join_add(self, left_plan_cost: PlanCost, right_plan_cost: PlanCost, execution_strategy: str = "parallel") -> PlanCost: """ Add the PlanCost objects for two joined plans (left_plan_cost and right_plan_cost) to the PlanCost object for the join operator. The execution strategy determines how the input times are combined. If the execution strategy is "parallel", the input time is the maximum of the two times. If the execution strategy is "sequential" (which is currently anything else), the input time is the sum of the two times. For quality, we compute the produce of the operator quality with the average of the two input qualities. NOTE: we currently assume the updating of the op_estimates are handled by the caller as there is not a universally correct meaning of addition of op_estimates. """ dct = {} for model_field in ["cost", "cost_lower_bound", "cost_upper_bound"]: op_field_value = getattr(self, model_field) left_plan_field_value = getattr(left_plan_cost, model_field) right_plan_field_value = getattr(right_plan_cost, model_field) if op_field_value is not None and left_plan_field_value is not None and right_plan_field_value is not None: dct[model_field] = op_field_value + left_plan_field_value + right_plan_field_value for model_field in ["time", "time_lower_bound", "time_upper_bound"]: op_field_value = getattr(self, model_field) left_plan_field_value = getattr(left_plan_cost, model_field) right_plan_field_value = getattr(right_plan_cost, model_field) if op_field_value is not None and left_plan_field_value is not None and right_plan_field_value is not None: if execution_strategy == "parallel": dct[model_field] = op_field_value + max(left_plan_field_value, right_plan_field_value) else: dct[model_field] = op_field_value + left_plan_field_value + right_plan_field_value for model_field in ["quality", "quality_lower_bound", "quality_upper_bound"]: op_field_value = getattr(self, model_field) left_plan_field_value = getattr(left_plan_cost, model_field) right_plan_field_value = getattr(right_plan_cost, model_field) if op_field_value is not None and left_plan_field_value is not None and right_plan_field_value is not None: dct[model_field] = op_field_value * ((left_plan_field_value + right_plan_field_value) / 2.0) return PlanCost(**dct) def __iadd__(self, other: PlanCost) -> PlanCost: """ NOTE: we currently assume the updating of the op_estimates are handled by the caller as there is not a universally correct meaning of addition of op_estimates. """ self.cost += other.cost self.time += other.time self.quality *= other.quality for model_field in ["cost_lower_bound", "cost_upper_bound", "time_lower_bound", "time_upper_bound"]: if getattr(self, model_field) is not None and getattr(other, model_field) is not None: summation = getattr(self, model_field) + getattr(other, model_field) setattr(self, model_field, summation) for model_field in ["quality_lower_bound", "quality_upper_bound"]: if getattr(self, model_field) is not None and getattr(other, model_field) is not None: product = getattr(self, model_field) * getattr(other, model_field) setattr(self, model_field, product) return self def __add__(self, other: PlanCost) -> PlanCost: """ NOTE: we currently assume the updating of the op_estimates are handled by the caller as there is not a universally correct meaning of addition of op_estimates. """ dct = { field: getattr(self, field) + getattr(other, field) for field in [ "cost", "cost_lower_bound", "cost_upper_bound", "time", "time_lower_bound", "time_upper_bound", ] } for model_field in ["quality", "quality_lower_bound", "quality_upper_bound"]: dct[model_field] = getattr(self, model_field) * getattr(other, model_field) return PlanCost(**dct) ================================================ FILE: src/palimpzest/policy.py ================================================ from __future__ import annotations import json from palimpzest.core.models import PlanCost def construct_policy_from_kwargs(**kwargs) -> Policy | None: """ Construct and return a policy object which is defined by the keyword arguments. This function accepts the following keyword arguments: - max_quality - min_cost - min_time - cost_budget - time_budget - quality_threshold If none of these keyword arguments are provided, the function will return None. """ # compute the number of objectives and constraints in the kwargs num_objectives = sum([bool(kwargs.get(key, False)) for key in ["max_quality", "min_cost", "min_time"]]) num_constraints = sum([bool(kwargs.get(key, False)) for key in ["cost_budget", "time_budget", "quality_threshold"]]) # if there are no policy kwargs provided, return None if num_objectives == 0 and num_constraints == 0: return None # Otherwise, assert that kwargs are valid assert num_objectives == 1, "Must optimize for one of max_quality, min_cost, or min_time." assert num_constraints <= 1, "Currently, PZ supports at most one constraint." # print warning if user tries to set a constraint and optimization goal on the same metric if "max_quality" in kwargs and "quality_threshold" in kwargs: print("Warning: Setting a constraint on quality and optimizing for quality is redundant.") if "min_cost" in kwargs and "cost_budget" in kwargs: print("Warning: Setting a constraint on cost and optimizing for cost is redundant.") if "min_time" in kwargs and "time_budget" in kwargs: print("Warning: Setting a constraint on time and optimizing for time is redundant.") # construct the policy object policy = None if "max_quality" in kwargs and "cost_budget" in kwargs: policy = MaxQualityAtFixedCost(kwargs["cost_budget"]) elif "max_quality" in kwargs and "time_budget" in kwargs: policy = MaxQualityAtFixedTime(kwargs["time_budget"]) elif "max_quality" in kwargs: policy = MaxQuality() elif "min_cost" in kwargs and "quality_threshold" in kwargs: policy = MinCostAtFixedQuality(kwargs["quality_threshold"]) elif "min_cost" in kwargs: policy = MinCost() elif "min_time" in kwargs and "quality_threshold" in kwargs: policy = MinTimeAtFixedQuality(kwargs["quality_threshold"]) elif "min_time" in kwargs: policy = MinTime() return policy class Policy: """ Base class for a policy. Each policy has two methods: constraint() and chooose(). The first method determines whether the given cost, runtime, and quality for a plan (or sub-plan) satisfy the policy's constraint(s). The second method takes in the (cost, runtime, quality) tuples for two plans (or subplans) and returns True if the first plan is better than the second one and False otherwise. """ def __init__(self): pass def get_primary_metric(self) -> str: """ Returns one of ["cost", "time", "quality"]; whichever corresponds to the maximization / minimization goal of the policy. Eventually we may make policies more general by allowing users to optimize some function: f(cost, time, quality). In that case, we may deprecate this method and update its callers. """ raise NotImplementedError("Calling this method from an abstract base class.") def get_dict(self) -> dict: """ Returns a dict representation of the policy which specifies how much weight (in [0,1]) should be given to each metric. """ raise NotImplementedError("Calling this method from an abstract base class.") def constraint(self, plan: PlanCost) -> bool: """ Return True if the given (cost, runtime, quality) for a plan (or subplan) satisfy the policy's constraint(s). Otherwise, return False. """ raise NotImplementedError("Calling this method from an abstract base class.") def choose(self, plan: PlanCost, other_plan: PlanCost) -> float: """ Return True if plan is better than other_plan and return False otherwise. """ raise NotImplementedError("Calling this method from an abstract base class.") def to_json_str(self) -> str: """Convert policy configuration to a JSON-serializable dictionary.""" return json.dumps({ "type": self.__class__.__name__, "config": self.get_dict() }, indent=2) class MaxQuality(Policy): """ This policy has no constraints and computes the best plan as the one with the higher quality. """ def __str__(self): return "Maximum Quality" def get_primary_metric(self) -> str: return "quality" def get_dict(self) -> dict: return {"cost": 0.0, "time": 0.0, "quality": 1.0} def constraint(self, plan: PlanCost) -> bool: """There is no constraint.""" return True def choose(self, plan: PlanCost, other_plan: PlanCost) -> float: """ Return True if plan has higher quality than other_plan and return False otherwise. Use cost and then runtime as tiebreakers. """ if plan.quality == other_plan.quality: if plan.cost == other_plan.cost: return plan.time < other_plan.time return plan.cost < other_plan.cost return plan.quality > other_plan.quality class MinCost(Policy): """ This policy has no constraints and computes the best plan as the one with the lower cost. """ def __str__(self): return "Minimum Cost" def get_primary_metric(self) -> str: return "cost" def get_dict(self) -> dict: return {"cost": 1.0, "time": 0.0, "quality": 0.0} def constraint(self, plan: PlanCost) -> bool: """There is no constraint.""" return True def choose(self, plan: PlanCost, other_plan: PlanCost) -> float: """ Return True if plan has lower cost than other_plan and return False otherwise. Use quality and then runtime as tiebreakers. """ if plan.cost == other_plan.cost: if plan.quality == other_plan.quality: return plan.time < other_plan.time return plan.quality > other_plan.quality return plan.cost < other_plan.cost class MinTime(Policy): """ This policy has no constraints and computes the best plan as the one with the lower runtime. """ def __str__(self): return "Minimum Time" def get_primary_metric(self) -> str: return "time" def get_dict(self) -> dict: return {"cost": 0.0, "time": 1.0, "quality": 0.0} def constraint(self, plan: PlanCost) -> bool: """There is no constraint.""" return True def choose(self, plan: PlanCost, other_plan: PlanCost) -> float: """ Return True if plan has lower runtime than other_plan and return False otherwise. Use quality and then cost as tiebreakers. """ if plan.time == other_plan.time: if plan.quality == other_plan.quality: return plan.cost < other_plan.cost return plan.quality > other_plan.quality return plan.time < other_plan.time class MaxQualityAtFixedCost(Policy): """ This policy applies a constraint (upper bound) on the cost of the plan and tries to maximize quality subject to that constraint. """ def __init__(self, max_cost: float): self.max_cost = max_cost def __str__(self): return "MaxQuality@FixedCost" def get_primary_metric(self) -> str: return "quality" def get_dict(self) -> dict: return {"cost": 0.5, "time": 0.0, "quality": 0.5} def constraint(self, plan: PlanCost) -> bool: return plan.cost < self.max_cost def choose(self, plan: PlanCost, other_plan: PlanCost) -> float: """ Return True if plan has higher quality than other_plan and return False otherwise. Use cost and then runtime as a tie-breaker. """ if plan.quality == other_plan.quality: if plan.cost == other_plan.cost: return plan.time < other_plan.time return plan.cost < other_plan.cost return plan.quality > other_plan.quality class MaxQualityAtFixedTime(Policy): """ This policy applies a constraint (upper bound) on the runtime of the plan and tries to maximize quality subject to that constraint. """ def __init__(self, max_time: float): self.max_time = max_time def __str__(self): return "MaxQuality@FixedTime" def get_primary_metric(self) -> str: return "quality" def get_dict(self) -> dict: return {"cost": 0.0, "time": 0.5, "quality": 0.5} def constraint(self, plan: PlanCost) -> bool: return plan.time < self.max_time def choose(self, plan: PlanCost, other_plan: PlanCost) -> float: """ Return True if plan has higher quality than other_plan and return False otherwise. Use runtime and then cost as a tie-breaker. """ if plan.quality == other_plan.quality: if plan.time == other_plan.time: return plan.cost < other_plan.cost return plan.time < other_plan.time return plan.quality > other_plan.quality class MinCostAtFixedQuality(Policy): """ This policy applies a constraint (lower bound) on the quality of the plan and tries to minimize cost subject to that constraint. """ def __init__(self, min_quality: float): self.min_quality = min_quality def __str__(self): return "MinCost@FixedQuality" def get_primary_metric(self) -> str: return "cost" def get_dict(self) -> dict: return {"cost": 0.5, "time": 0.0, "quality": 0.5} def constraint(self, plan: PlanCost) -> bool: return plan.quality > self.min_quality def choose(self, plan: PlanCost, other_plan: PlanCost) -> float: """ Return True if plan has lower cost than other_plan and return False otherwise. Use quality and then runtime as a tie-breaker. """ if plan.cost == other_plan.cost: if plan.quality == other_plan.quality: return plan.time < other_plan.time return plan.quality > other_plan.quality return plan.cost < other_plan.cost class MinTimeAtFixedQuality(Policy): """ This policy applies a constraint (lower bound) on the quality of the plan and tries to minimize runtime subject to that constraint. """ def __init__(self, min_quality: float): self.min_quality = min_quality def __str__(self): return "MinTime@FixedQuality" def get_primary_metric(self) -> str: return "time" def get_dict(self) -> dict: return {"cost": 0.0, "time": 0.5, "quality": 0.5} def constraint(self, plan: PlanCost) -> bool: return plan.quality > self.min_quality def choose(self, plan: PlanCost, other_plan: PlanCost) -> float: """ Return True if plan has lower runtime than other_plan and return False otherwise. Use quality and then cost as a tie-breaker. """ if plan.time == other_plan.time: if plan.quality == other_plan.quality: return plan.cost < other_plan.cost return plan.quality > other_plan.quality return plan.time < other_plan.time ================================================ FILE: src/palimpzest/prompts/__init__.py ================================================ from palimpzest.prompts.agent_prompts import ( CODE_AGENT_SYSTEM_PROMPT, DATA_DISCOVERY_AGENT_INITIAL_PLAN_PROMPT, DATA_DISCOVERY_AGENT_REPORT_PROMPT, DATA_DISCOVERY_AGENT_TASK_PROMPT, DATA_DISCOVERY_AGENT_UPDATE_PLAN_POST_MESSAGES_PROMPT, DATA_DISCOVERY_AGENT_UPDATE_PLAN_PRE_MESSAGES_PROMPT, FINAL_ANSWER_POST_MESSAGES_PROMPT, FINAL_ANSWER_PRE_MESSAGES_PROMPT, ) from palimpzest.prompts.context_search import CONTEXT_SEARCH_PROMPT from palimpzest.prompts.prompt_factory import PromptFactory from palimpzest.prompts.prompt_manager import PromptManager from palimpzest.prompts.utils import ( ONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION, ONE_TO_ONE_OUTPUT_FORMAT_INSTRUCTION, ) from palimpzest.prompts.validator import ( FLAT_MAP_IMAGE_VALIDATOR_PROMPT, FLAT_MAP_VALIDATOR_PROMPT, MAP_IMAGE_VALIDATOR_PROMPT, MAP_VALIDATOR_PROMPT, RETRIEVE_VALIDATOR_PROMPT, ) __all__ = [ # agent prompts "CODE_AGENT_SYSTEM_PROMPT", "DATA_DISCOVERY_AGENT_INITIAL_PLAN_PROMPT", "DATA_DISCOVERY_AGENT_REPORT_PROMPT", "DATA_DISCOVERY_AGENT_TASK_PROMPT", "DATA_DISCOVERY_AGENT_UPDATE_PLAN_POST_MESSAGES_PROMPT", "DATA_DISCOVERY_AGENT_UPDATE_PLAN_PRE_MESSAGES_PROMPT", "FINAL_ANSWER_POST_MESSAGES_PROMPT", "FINAL_ANSWER_PRE_MESSAGES_PROMPT", # context search "CONTEXT_SEARCH_PROMPT", # prompt cache "PromptManager", # prompt factory "PromptFactory", # utils "ONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION", "ONE_TO_ONE_OUTPUT_FORMAT_INSTRUCTION", # validator "FLAT_MAP_IMAGE_VALIDATOR_PROMPT", "FLAT_MAP_VALIDATOR_PROMPT", "MAP_IMAGE_VALIDATOR_PROMPT", "MAP_VALIDATOR_PROMPT", "RETRIEVE_VALIDATOR_PROMPT", ] ================================================ FILE: src/palimpzest/prompts/agent_prompts.py ================================================ CODE_AGENT_SYSTEM_PROMPT = """You are an expert assistant who can solve any task using code blobs. You will be given a task to solve as best you can. To do so, you have been given access to a list of tools: these tools are basically Python functions which you can call with code. To solve the task, you must plan forward to proceed in a series of steps, in a cycle of 'Thought:', '', and 'Observation:' sequences. At each step, in the 'Thought:' sequence, you should first explain your reasoning towards solving the task and the tools that you want to use. Then in the '' sequence, you should write the code in simple Python. The code sequence must end with '' sequence. During each intermediate step, you can use 'print()' to save whatever important information you will then need. These print outputs will then appear in the 'Observation:' field, which will be available as input for the next step. In the end you have to return a final answer using the `final_answer` tool. Here are a few examples using notional tools: --- Task: "Generate an image of the oldest person in this document." Thought: I will proceed step by step and use the following tools: `document_qa` to find the oldest person in the document, then `image_generator` to generate an image according to the answer. answer = document_qa(document=document, question="Who is the oldest person mentioned?") print(answer) Observation: "The oldest person in the document is John Doe, a 55 year old lumberjack living in Newfoundland." Thought: I will now generate an image showcasing the oldest person. image = image_generator("A portrait of John Doe, a 55-year-old man living in Canada.") final_answer(image) --- Task: "What is the result of the following operation: 5 + 3 + 1294.678?" Thought: I will use python code to compute the result of the operation and then return the final answer using the `final_answer` tool result = 5 + 3 + 1294.678 final_answer(result) --- Task: "Answer the question in the variable `question` about the image stored in the variable `image`. The question is in French. You have been provided with these additional arguments, that you can access using the keys as variables in your python code: {'question': 'Quel est l'animal sur l'image?', 'image': 'path/to/image.jpg'}" Thought: I will use the following tools: `translator` to translate the question into English and then `image_qa` to answer the question on the input image. translated_question = translator(question=question, src_lang="French", tgt_lang="English") print(f"The translated question is {translated_question}.") answer = image_qa(image=image, question=translated_question) final_answer(f"The answer is {answer}") --- Task: In a 1979 interview, Stanislaus Ulam discusses with Martin Sherwin about other great physicists of his time, including Oppenheimer. What does he say was the consequence of Einstein learning too much math on his creativity, in one word? Thought: I need to find and read the 1979 interview of Stanislaus Ulam with Martin Sherwin. pages = web_search(query="1979 interview Stanislaus Ulam Martin Sherwin physicists Einstein") print(pages) Observation: No result found for query "1979 interview Stanislaus Ulam Martin Sherwin physicists Einstein". Thought: The query was maybe too restrictive and did not find any results. Let's try again with a broader query. pages = web_search(query="1979 interview Stanislaus Ulam") print(pages) Observation: Found 6 pages: [Stanislaus Ulam 1979 interview](https://ahf.nuclearmuseum.org/voices/oral-histories/stanislaus-ulams-interview-1979/) [Ulam discusses Manhattan Project](https://ahf.nuclearmuseum.org/manhattan-project/ulam-manhattan-project/) (truncated) Thought: I will read the first 2 pages to know more. for url in ["https://ahf.nuclearmuseum.org/voices/oral-histories/stanislaus-ulams-interview-1979/", "https://ahf.nuclearmuseum.org/manhattan-project/ulam-manhattan-project/"]: whole_page = visit_webpage(url) print(whole_page) print("\n" + "="*80 + "\n") # Print separator between pages Observation: Manhattan Project Locations: Los Alamos, NM Stanislaus Ulam was a Polish-American mathematician. He worked on the Manhattan Project at Los Alamos and later helped design the hydrogen bomb. In this interview, he discusses his work at (truncated) Thought: I now have the final answer: from the webpages visited, Stanislaus Ulam says of Einstein: "He learned too much mathematics and sort of diminished, it seems to me personally, it seems to me his purely physics creativity." Let's answer in one word. final_answer("diminished") --- Task: "Which city has the highest population: Guangzhou or Shanghai?" Thought: I need to get the populations for both cities and compare them: I will use the tool `web_search` to get the population of both cities. for city in ["Guangzhou", "Shanghai"]: print(f"Population {city}:", web_search(f"{city} population") Observation: Population Guangzhou: ['Guangzhou has a population of 15 million inhabitants as of 2021.'] Population Shanghai: '26 million (2019)' Thought: Now I know that Shanghai has the highest population. final_answer("Shanghai") --- Task: "What is the current age of the pope, raised to the power 0.36?" Thought: I will use the tool `wikipedia_search` to get the age of the pope, and confirm that with a web search. pope_age_wiki = wikipedia_search(query="current pope age") print("Pope age as per wikipedia:", pope_age_wiki) pope_age_search = web_search(query="current pope age") print("Pope age as per google search:", pope_age_search) Observation: Pope age: "The pope Francis is currently 88 years old." Thought: I know that the pope is 88 years old. Let's compute the result using python code. pope_current_age = 88 ** 0.36 final_answer(pope_current_age) Above example were using notional tools that might not exist for you. On top of performing computations in the Python code snippets that you create, you only have access to these tools, behaving like regular python functions: ```python {%- for tool in tools.values() %} def {{ tool.name }}({% for arg_name, arg_info in tool.inputs.items() %}{{ arg_name }}: {{ arg_info.type }}{% if not loop.last %}, {% endif %}{% endfor %}) -> {{tool.output_type}}: \"\"\"{{ tool.description }} Args: {%- for arg_name, arg_info in tool.inputs.items() %} {{ arg_name }}: {{ arg_info.description }} {%- endfor %} \"\"\" {% endfor %} ``` {%- if managed_agents and managed_agents.values() | list %} You can also give tasks to team members. Calling a team member works similarly to calling a tool: provide the task description as the 'task' argument. Since this team member is a real human, be as detailed and verbose as necessary in your task description. You can also include any relevant variables or context using the 'additional_args' argument. Here is a list of the team members that you can call: ```python {%- for agent in managed_agents.values() %} def {{ agent.name }}(task: str, additional_args: dict[str, Any]) -> str: \"\"\"{{ agent.description }} Args: task: Long detailed description of the task. additional_args: Dictionary of extra inputs to pass to the managed agent, e.g. images, dataframes, or any other contextual data it may need. \"\"\" {% endfor %} ``` {%- endif %} Here are the rules you should always follow to solve your task: 1. Always provide a 'Thought:' sequence, and a '' sequence ending with '', else you will fail. 2. Use only variables that you have defined! 3. Always use the right arguments for the tools. DO NOT pass the arguments as a dict as in 'answer = wikipedia_search({'query': "What is the place where James Bond lives?"})', but use the arguments directly as in 'answer = wikipedia_search(query="What is the place where James Bond lives?")'. 4. Take care to not chain too many sequential tool calls in the same code block, especially when the output format is unpredictable. For instance, a call to wikipedia_search has an unpredictable return format, so do not have another tool call that depends on its output in the same block: rather output results with print() to use them in the next block. 5. Call a tool only when needed, and never re-do a tool call that you previously did with the exact same parameters. 6. Don't name any new variable with the same name as a tool: for instance don't name a variable 'final_answer'. 7. Never create any notional variables in our code, as having these in your logs will derail you from the true variables. 8. You can use imports in your code, but only from the following list of modules: {{authorized_imports}} 9. The state persists between code executions: so if in one step you've created variables or imported modules, these will all persist. 10. Don't give up! You're in charge of solving the task, not providing directions to solve it. {%- if custom_instructions %} {{custom_instructions}} {%- endif %} Now Begin!""" DATA_DISCOVERY_AGENT_INITIAL_PLAN_PROMPT = """You are a world expert at analyzing a situation to derive facts, and plan accordingly towards solving a task. Below I will present you a task. You will need to 1. build a survey of facts known or needed to solve the task, then 2. make a plan of action to solve the task. ## 1. Facts survey You will build a comprehensive preparatory survey of which facts we have at our disposal and which ones we still need. These "facts" will typically be specific names, dates, values, etc. Be sure to report the facts you look up and derive, as well as the sources where you find these facts. Your answer should use the below headings: ### 1.1. Facts given in the task List here the specific facts given in the task that could help you (there might be nothing here). ### 1.2. Facts to look up List here any facts that we may need to look up. Also list where to find each of these, for instance a website, a file... - maybe the task contains some sources that you should re-use here. ### 1.3. Facts to derive List here anything that we want to derive from the above by logical reasoning, for instance computation or simulation. ### 1.4. Fact Sources List the source of each fact that you have looked up or derived. The source may be a file, a database table, a web page, etc. Be sure that someone can use these sources to reproduce your work. Don't make any assumptions. For each item, provide a thorough reasoning. Do not add anything else on top of the four headings above. ## 2. Plan Then for the given task, develop a step-by-step high-level plan taking into account the above inputs and list of facts. This plan should involve individual tasks based on the available tools, that if executed correctly will yield the correct answer. Do not skip steps, do not add any superfluous steps. Only write the high-level plan, DO NOT DETAIL INDIVIDUAL TOOL CALLS. After writing the final step of the plan, write the '' tag and stop there. You can leverage these tools, behaving like regular python functions: ```python {%- for tool in tools.values() %} def {{ tool.name }}({% for arg_name, arg_info in tool.inputs.items() %}{{ arg_name }}: {{ arg_info.type }}{% if not loop.last %}, {% endif %}{% endfor %}) -> {{tool.output_type}}: \"\"\"{{ tool.description }} Args: {%- for arg_name, arg_info in tool.inputs.items() %} {{ arg_name }}: {{ arg_info.description }} {%- endfor %} \"\"\" {% endfor %} ``` The tools you have been given will provide you with access to a dataset with the following description: Context: {{context_description}}\n {%- if managed_agents and managed_agents.values() | list %} You can also give tasks to team members. Calling a team member works similarly to calling a tool: provide the task description as the 'task' argument. Since this team member is a real human, be as detailed and verbose as necessary in your task description. You can also include any relevant variables or context using the 'additional_args' argument. Here is a list of the team members that you can call: ```python {%- for agent in managed_agents.values() %} def {{ agent.name }}(task: str, additional_args: dict[str, Any]) -> str: \"\"\"{{ agent.description }} Args: task: Long detailed description of the task. additional_args: Dictionary of extra inputs to pass to the managed agent, e.g. images, dataframes, or any other contextual data it may need. \"\"\" {% endfor %} ``` {%- endif %} --- Now begin! Here is your task: ``` search: {{task}} ``` First in part 1, write the facts survey, then in part 2, write your plan. """ DATA_DISCOVERY_AGENT_UPDATE_PLAN_PRE_MESSAGES_PROMPT = """You are a world expert at analyzing a situation, and plan accordingly towards solving a task. You have been given the following task: ``` search: {{task}} ``` You are working with the following dataset: ``` context: {{context_description}} ``` Below you will find a history of attempts made to solve this task. You will first have to produce a survey of known and unknown facts, then propose a step-by-step high-level plan to solve the task. If the previous tries so far have met some success, your updated plan can build on these results. If you are stalled, you can make a completely new plan starting from scratch. Find the task and history below: """ DATA_DISCOVERY_AGENT_UPDATE_PLAN_POST_MESSAGES_PROMPT = """Now write your updated facts below, taking into account the above history: ## 1. Updated facts survey ### 1.1. Facts given in the task ### 1.2. Facts that we have learned ### 1.3. Facts still to look up ### 1.4. Facts still to derive ### 1.5. Fact Sources Then write a step-by-step high-level plan to solve the task above. ## 2. Plan ### 2. 1. ... Etc. This plan should involve individual tasks based on the available tools, that if executed correctly will yield the correct answer. Beware that you have {remaining_steps} steps remaining. Do not skip steps, do not add any superfluous steps. Only write the high-level plan, DO NOT DETAIL INDIVIDUAL TOOL CALLS. After writing the final step of the plan, write the '' tag and stop there. You can leverage these tools, behaving like regular python functions: ```python {%- for tool in tools.values() %} def {{ tool.name }}({% for arg_name, arg_info in tool.inputs.items() %}{{ arg_name }}: {{ arg_info.type }}{% if not loop.last %}, {% endif %}{% endfor %}) -> {{tool.output_type}}: \"\"\"{{ tool.description }} Args: {%- for arg_name, arg_info in tool.inputs.items() %} {{ arg_name }}: {{ arg_info.description }} {%- endfor %}\"\"\" {% endfor %} ``` The tools you have been given will provide you with access to a dataset with the following description: Context: {{context_description}} {%- if managed_agents and managed_agents.values() | list %} You can also give tasks to team members. Calling a team member works similarly to calling a tool: provide the task description as the 'task' argument. Since this team member is a real human, be as detailed and verbose as necessary in your task description. You can also include any relevant variables or context using the 'additional_args' argument. Here is a list of the team members that you can call: ```python {%- for agent in managed_agents.values() %} def {{ agent.name }}(task: str, additional_args: dict[str, Any]) -> str: \"\"\"{{ agent.description }} Args: task: Long detailed description of the task. additional_args: Dictionary of extra inputs to pass to the managed agent, e.g. images, dataframes, or any other contextual data it may need. \"\"\" {% endfor %} ``` {%- endif %} Now write your updated facts survey below, then your new plan. """ DATA_DISCOVERY_AGENT_TASK_PROMPT = """You're a helpful agent named '{{name}}'. You have been submitted this task by your manager. --- Task: {{task}} --- You have also been given access to tools which will help you navigate a dataset with the following description: --- Context: {{context_description}} --- You're helping your manager solve a search task: so make sure to not provide a one-line answer, but give as much information as possible to give them a clear understanding of the answer and the context in which you produced it. In particular, be sure to report what source(s) you use and what the contents of those source(s) contain, even if they are irrelevant for solving this task. Your final_answer WILL HAVE to contain these parts: ### 1. Task outcome (short version): ### 2. Task outcome (extremely detailed version): ### 3. Additional context (if relevant): Put all these in your final_answer tool, everything that you do not pass as an argument to final_answer will be lost. And even if your task resolution is not successful, please return as much context as possible, so that your manager can act upon this feedback. """ DATA_DISCOVERY_AGENT_REPORT_PROMPT = """Here is the final answer from your managed agent '{{name}}': {{final_answer}}""" FINAL_ANSWER_PRE_MESSAGES_PROMPT = """An agent tried to answer a user query but it got stuck and failed to do so. You are tasked with providing an answer instead. Here is the agent's memory:""" FINAL_ANSWER_POST_MESSAGES_PROMPT = """Based on the above, please provide an answer to the following user task: {{task}} """ ================================================ FILE: src/palimpzest/prompts/aggregate_prompts.py ================================================ """This file contains prompts for aggregation operations.""" ### BASE PROMPTS ### AGG_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and an output field to generate. Your task is to generate a JSON object which aggregates the input and fills in the output field with the correct value. You will be provided with a description of each input field and each output field. The field in the output JSON object can be derived using information from the context. {output_format_instruction} Finish your response with a newline character followed by --- An example is shown below: --- INPUT FIELDS: {example_input_fields} OUTPUT FIELDS: {example_output_fields} CONTEXT: {{{example_context}}} {{{second_example_context}}} {{{third_example_context}}}{image_disclaimer}{audio_disclaimer} AGGREGATION INSTRUCTION: {example_agg_instruction} Let's think step-by-step in order to answer the question. REASONING: {example_reasoning} ANSWER: {{{example_answer}}} --- """ AGG_NO_REASONING_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and an output field to generate. Your task is to generate a JSON object which aggregates the input and fills in the output field with the correct value. You will be provided with a description of each input field and each output field. The field in the output JSON object can be derived using information from the context. {output_format_instruction} Finish your response with a newline character followed by --- An example is shown below: --- INPUT FIELDS: {example_input_fields} OUTPUT FIELDS: {example_output_fields} CONTEXT: {{{example_context}}} {{{second_example_context}}} {{{third_example_context}}}{image_disclaimer}{audio_disclaimer} AGGREGATION INSTRUCTION: {example_agg_instruction} ANSWER: {{{example_answer}}} --- """ AGG_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and an output field to generate. Your task is to generate a JSON object which aggregates the input and fills in the output field with the correct value. You will be provided with a description of each input field and each output field. The field in the output JSON object can be derived using information from the context. {desc_section} {output_format_instruction} Finish your response with a newline character followed by --- --- INPUT FIELDS: {input_fields_desc} OUTPUT FIELDS: {output_fields_desc} CONTEXT: {context}<> AGGREGATION INSTRUCTION: {agg_instruction} Let's think step-by-step in order to answer the question. REASONING: """ AGG_NO_REASONING_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and an output field to generate. Your task is to generate a JSON object which aggregates the input and fills in the output field with the correct value. You will be provided with a description of each input field and each output field. The field in the output JSON object can be derived using information from the context. {desc_section} {output_format_instruction} Finish your response with a newline character followed by --- --- INPUT FIELDS: {input_fields_desc} OUTPUT FIELDS: {output_fields_desc} CONTEXT: {context}<> AGGREGATION INSTRUCTION: {agg_instruction} ANSWER: """ ================================================ FILE: src/palimpzest/prompts/context_search.py ================================================ CONTEXT_SEARCH_PROMPT = """You are a helpful agent whose job is to propose search queries that will assist in finding information that is relevant to performing a computation. Please propose a concise sentence which will be used to search for relevant information for the following computation instruction. INSTRUCTION: {instruction} SEARCH QUERY: """ ================================================ FILE: src/palimpzest/prompts/convert_prompts.py ================================================ """This file contains prompts for convert operations.""" ### BASE PROMPTS ### MAP_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a set of output fields to generate. Your task is to generate a JSON object which fills in the output fields with the correct values. You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the context. {output_format_instruction} Finish your response with a newline character followed by --- An example is shown below: --- INPUT FIELDS: {example_input_fields} OUTPUT FIELDS: {example_output_fields} CONTEXT: {{{example_context}}}{image_disclaimer}{audio_disclaimer} Let's think step-by-step in order to answer the question. REASONING: {example_reasoning} ANSWER: {{{example_answer}}} --- """ MAP_NO_REASONING_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a set of output fields to generate. Your task is to generate a JSON object which fills in the output fields with the correct values. You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the context. {output_format_instruction} Finish your response with a newline character followed by --- An example is shown below: --- INPUT FIELDS: {example_input_fields} OUTPUT FIELDS: {example_output_fields} CONTEXT: {{{example_context}}}{image_disclaimer}{audio_disclaimer} ANSWER: {{{example_answer}}} --- """ MAP_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a set of output fields to generate. Your task is to generate a JSON object which fills in the output fields with the correct values. You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the context. {desc_section} {output_format_instruction} Finish your response with a newline character followed by --- --- INPUT FIELDS: {input_fields_desc} OUTPUT FIELDS: {output_fields_desc} <>CONTEXT: {context}<> Let's think step-by-step in order to answer the question. REASONING: """ MAP_NO_REASONING_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a set of output fields to generate. Your task is to generate a JSON object which fills in the output fields with the correct values. You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the context. {desc_section} {output_format_instruction} Finish your response with a newline character followed by --- --- INPUT FIELDS: {input_fields_desc} OUTPUT FIELDS: {output_fields_desc} <>CONTEXT: {context}<> ANSWER: """ ================================================ FILE: src/palimpzest/prompts/critique_and_refine_prompts.py ================================================ """This file contains prompts for CritiqueAndRefineConvert operations.""" ### CRITIQUE PROMPT AND CRITERIA ### BASE_CRITIQUE_PROMPT = """You are a helpful assistant tasked with critiquing the output of a model based on a given prompt. Below is the original user prompt used to generate the output: ORIGINAL PROMPT: <> Here is the output generated by the model: OUTPUT: {original_output} Your task is to critique the output based on the following: {critique_criteria} {finish_instruction} """ MAP_CRITIQUE_CRITERIA = """1. Does the JSON object adhere to the required format? Highlight any structural issues. 2. Are the values of the output fields accurate based on the provided context? If any fields are incorrect or missing, provide specific examples. 3. Are there any logical errors in reasoning used to derive the output? Provide detailed feedback on potential mistakes. """ FILTER_CRITIQUE_CRITERIA = """1. Does the output adhere to the required TRUE/FALSE format? Highlight any issues. 2. Is the TRUE/FALSE determination accurate based on the provided context? If there is evidence for an incorrect determination, provide specific reasons why. 3. Are there any logical errors in reasoning used to derive the TRUE/FALSE determination? Provide detailed feedback on potential mistakes. """ MAP_CRITIQUE_FINISH_INSTRUCTION = """Finish your critique with actionable recommendations for improving the JSON object.""" FILTER_CRITIQUE_FINISH_INSTRUCTION = """Finish your critique with actionable recommendations for improving the model's reasoning and answer.""" ### REFINEMENT PROMPT AND CRITERIA ### BASE_REFINEMENT_PROMPT = """You are a helpful assistant tasked with refining the output of a model based on a critique. Below is the original user prompt used to generate the output: ORIGINAL PROMPT: <> Here is the original output generated by the model: OUTPUT: {original_output} Here is the critique of the output: CRITIQUE: {critique_output} Your task is to refine the original output to address the critique. Ensure that: {refinement_criteria} {finish_instruction} """ MAP_REFINEMENT_CRITERIA = """1. The answer adheres to the required JSON format specified in the original prompt. 2. Correctly derives all values for the output fields based on the provided context. 3. Resolves any logical errors identified in the critique. """ FILTER_REFINEMENT_CRITERIA = """1. The answer adheres to the required TRUE/FALSE format specified in the original prompt. 2. Correctly derives the final TRUE/FALSE determination based on the provided context. 3. Resolves any logical errors identified in the critique. """ MAP_REFINEMENT_FINISH_INSTRUCTION = """Return the refined JSON object as your final answer.""" FILTER_REFINEMENT_FINISH_INSTRUCTION = """Return the final TRUE/FALSE answer.""" ================================================ FILE: src/palimpzest/prompts/filter_prompts.py ================================================ """This file contains prompts for filter operations.""" ### BASE PROMPTS ### FILTER_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a filter condition. Output TRUE if the context satisfies the filter condition, and FALSE otherwise. Remember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by --- An example is shown below: --- INPUT FIELDS: {example_input_fields} FILTER CONDITION: {example_filter_condition} CONTEXT: {{{example_context}}}{image_disclaimer}{audio_disclaimer} Let's think step-by-step in order to answer the question. REASONING: {example_reasoning} ANSWER: TRUE --- """ FILTER_NO_REASONING_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a filter condition. Output TRUE if the context satisfies the filter condition, and FALSE otherwise. Remember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by --- An example is shown below: --- INPUT FIELDS: {example_input_fields} FILTER CONDITION: {example_filter_condition} CONTEXT: {{{example_context}}}{image_disclaimer}{audio_disclaimer} ANSWER: TRUE --- """ FILTER_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a filter condition. Output TRUE if the context satisfies the filter condition, and FALSE otherwise. {desc_section} Remember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by --- --- INPUT FIELDS: {input_fields_desc} FILTER CONDITION: {filter_condition} <>CONTEXT: {context}<> Let's think step-by-step in order to answer the question. REASONING: """ FILTER_NO_REASONING_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a filter condition. Output TRUE if the context satisfies the filter condition, and FALSE otherwise. {desc_section} Remember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by --- --- INPUT FIELDS: {input_fields_desc} FILTER CONDITION: {filter_condition} <>CONTEXT: {context}<> ANSWER: """ ================================================ FILE: src/palimpzest/prompts/join_prompts.py ================================================ """This file contains prompts for join operations.""" ### BASE PROMPTS ### JOIN_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with two data records and a join condition. Output TRUE if the two data records satisfy the join condition, and FALSE otherwise. Remember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by --- An example is shown below: --- LEFT INPUT FIELDS: {example_input_fields} RIGHT INPUT FIELDS: {right_example_input_fields} JOIN CONDITION: {example_join_condition} LEFT CONTEXT: {{{example_context}}}{image_disclaimer}{audio_disclaimer} RIGHT CONTEXT: {{{right_example_context}}}{right_image_disclaimer}{right_audio_disclaimer} Let's think step-by-step in order to evaluate the join condition. REASONING: {example_reasoning} ANSWER: TRUE --- """ JOIN_NO_REASONING_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with two data records and a join condition. Output TRUE if the two data records satisfy the join condition, and FALSE otherwise. Remember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by --- An example is shown below: --- LEFT INPUT FIELDS: {example_input_fields} RIGHT INPUT FIELDS: {right_example_input_fields} JOIN CONDITION: {example_join_condition} LEFT CONTEXT: {{{example_context}}}{image_disclaimer}{audio_disclaimer} RIGHT CONTEXT: {{{right_example_context}}}{right_image_disclaimer}{right_audio_disclaimer} ANSWER: TRUE --- """ JOIN_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with two data records and a join condition. Output TRUE if the two data records satisfy the join condition, and FALSE otherwise. {desc_section} Remember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by --- --- LEFT INPUT FIELDS: {input_fields_desc} RIGHT INPUT FIELDS: {right_input_fields_desc} JOIN CONDITION: {join_condition} <>LEFT CONTEXT: {context}<> RIGHT CONTEXT: {right_context}<> Let's think step-by-step in order to evaluate the join condition. REASONING: """ JOIN_NO_REASONING_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with two data records and a join condition. Output TRUE if the two data records satisfy the join condition, and FALSE otherwise. {desc_section} Remember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by --- --- LEFT INPUT FIELDS: {input_fields_desc} RIGHT INPUT FIELDS: {right_input_fields_desc} JOIN CONDITION: {join_condition} <>LEFT CONTEXT: {context}<> RIGHT CONTEXT: {right_context}<> ANSWER: """ ================================================ FILE: src/palimpzest/prompts/moa_aggregator_prompts.py ================================================ """This file contains prompts for Mixture-of-Agents aggregator operations.""" ### SYSTEM PROMPTS ### MAP_MOA_AGG_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to generate a JSON object. You will be presented with one or more outputs produced by a set of models. Your task is to synthesize these responses into a single, high-quality JSON object which fills in the output fields with the correct values. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect. You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the model responses. {output_format_instruction} Finish your response with a newline character followed by --- An example is shown below: --- MODEL RESPONSE 1: the text mentions the scientist's full name "Augusta Ada King, Countess of Lovelace" and states she was an English mathematician who worked on Babbage's Analytical Engine. MODEL RESPONSE 2: the text passage mentions the scientist's name as "Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace" and the scientist's birthday as "December 10, 1815". Therefore, the name of the scientist is "Augusta Ada King" and the birth year is 1815. INPUT FIELDS: - text: a text passage describing a scientist - birthday: the scientist's birthday OUTPUT FIELDS: - name: the name of the scientist - birth_year: the year the scientist was born Let's think step-by-step in order to answer the question. REASONING: Looking at both model responses, they agree that the scientist's formal name is "Augusta Ada King". Model Response 2 correctly extracts the birth year from the birthday field as 1815. The responses are consistent and provide sufficient evidence for these values. ANSWER: {{ "name": "Augusta Ada King", "birth_year": 1815 }} --- """ FILTER_MOA_AGG_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to answer a TRUE/FALSE question. You will be presented with one or more outputs produced by a set of models. Your task is to synthesize these responses into a single TRUE/FALSE answer. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect. You will also be provided with a description of each input field and the filter condition. Remember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by --- An example is shown below: --- MODEL RESPONSE 1: The context describes Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace, who is widely recognized as a foundational figure in computer science. Therefore, the answer is TRUE. MODEL RESPONSE 2: Based on the context provided, Ada Lovelace is indeed a foundational computer scientist, therefore the answer is TRUE. INPUT FIELDS: - text: a text passage describing a scientist - birthday: the scientist's birthday - image: an image of the scientist - recording: an audio recording of a newscast about the scientist's contributions to their field FILTER CONDITION: The subject of the input is a foundational computer scientist. Let's think step-by-step in order to answer the question. REASONING: Both model responses agree that the context describes Ada Lovelace, who is widely recognized as a foundational figure in computer science. The evidence from the text passage supports this conclusion. ANSWER: TRUE --- """ ### USER / INSTANCE-SPECIFIC PROMPTS ### MAP_MOA_AGG_BASE_USER_PROMPT = """You are a helpful assistant whose job is to generate a JSON object. You will be presented with one or more outputs produced by a set of models. Your task is to synthesize these responses into a single, high-quality JSON object which fills in the output fields with the correct values. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect. You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the model responses. {output_format_instruction} Finish your response with a newline character followed by --- --- INPUT FIELDS: {input_fields_desc} OUTPUT FIELDS: {output_fields_desc} <>{model_responses} Let's think step-by-step in order to answer the question. REASONING: """ FILTER_MOA_AGG_BASE_USER_PROMPT = """You are a helpful assistant whose job is to answer a TRUE/FALSE question. You will be presented with one or more outputs produced by a set of models. Your task is to synthesize these responses into a single TRUE/FALSE answer. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect. You will also be provided with a description of each input field and the filter condition. Remember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by --- --- INPUT FIELDS: {input_fields_desc} FILTER CONDITION: {filter_condition} <>{model_responses} Let's think step-by-step in order to answer the question. REASONING: """ ================================================ FILE: src/palimpzest/prompts/moa_proposer_prompts.py ================================================ """This file contains prompts for MixtureOfAgentsConvert operations.""" ### SYSTEM PROMPTS ### MAP_MOA_PROPOSER_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a set of output fields to generate. Your task is to generate a detailed and succinct analysis describing what you believe is the correct value for each output field. Be sure to cite information from the context as evidence of why your answers are correct. Do not hallucinate evidence. You will be provided with a description of each input field and each output field. An example is shown below: --- INPUT FIELDS: {example_input_fields} OUTPUT FIELDS: {example_output_fields} CONTEXT: {{{example_context}}}{image_disclaimer}{audio_disclaimer} Let's think step-by-step in order to answer the question. ANSWER: {example_answer} --- """ FILTER_MOA_PROPOSER_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a filter condition. Your task is to generate a detailed and succinct analysis describing whether you believe the input satisfies the filter condition. Be sure to cite information from the context as evidence of why your determination is correct. Do not hallucinate evidence. You will be provided with a description of each input field. An example is shown below: --- INPUT FIELDS: {example_input_fields} FILTER CONDITION: {example_filter_condition} CONTEXT: {{{example_context}}}{image_disclaimer}{audio_disclaimer} Let's think step-by-step in order to answer the question. ANSWER: {example_answer} --- """ ### USER / INSTANCE-SPECIFIC PROMPTS ### MAP_MOA_PROPOSER_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a set of output fields to generate. Your task is to generate a detailed and succinct analysis describing what you believe is the correct value for each output field. Be sure to cite information from the context as evidence of why your answers are correct. Do not hallucinate evidence. {desc_section} You will be provided with a description of each input field and each output field. --- INPUT FIELDS: {input_fields_desc} OUTPUT FIELDS: {output_fields_desc} <>CONTEXT: {context}<> Let's think step-by-step in order to answer the question. ANSWER: """ FILTER_MOA_PROPOSER_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a filter condition. Your task is to generate a detailed and succinct analysis describing whether you believe the input satisfies the filter condition. Be sure to cite information from the context as evidence of why your determination is correct. Do not hallucinate evidence. {desc_section} You will be provided with a description of each input field. An example is shown below: --- INPUT FIELDS: {input_fields_desc} FILTER CONDITION: {filter_condition} <>CONTEXT: {context}<> Let's think step-by-step in order to answer the question. ANSWER: """ ================================================ FILE: src/palimpzest/prompts/prompt_factory.py ================================================ """This file contains factory methods which return template prompts and return messages for chat payloads.""" import base64 import json import os from typing import Any from pydantic import BaseModel from palimpzest.constants import ( LLAMA_CONTEXT_TOKENS_LIMIT, TOKENS_PER_CHARACTER, Cardinality, Modality, Model, PromptStrategy, ) from palimpzest.core.elements.records import DataRecord from palimpzest.core.lib.schemas import ( AUDIO_FIELD_TYPES, IMAGE_FIELD_TYPES, AudioBase64, AudioFilepath, ImageBase64, ImageFilepath, ImageURL, ) from palimpzest.prompts.aggregate_prompts import ( AGG_BASE_SYSTEM_PROMPT, AGG_BASE_USER_PROMPT, AGG_NO_REASONING_BASE_SYSTEM_PROMPT, AGG_NO_REASONING_BASE_USER_PROMPT, ) from palimpzest.prompts.convert_prompts import ( MAP_BASE_SYSTEM_PROMPT, MAP_BASE_USER_PROMPT, MAP_NO_REASONING_BASE_SYSTEM_PROMPT, MAP_NO_REASONING_BASE_USER_PROMPT, ) from palimpzest.prompts.critique_and_refine_prompts import ( BASE_CRITIQUE_PROMPT, BASE_REFINEMENT_PROMPT, FILTER_CRITIQUE_CRITERIA, FILTER_CRITIQUE_FINISH_INSTRUCTION, FILTER_REFINEMENT_CRITERIA, FILTER_REFINEMENT_FINISH_INSTRUCTION, MAP_CRITIQUE_CRITERIA, MAP_CRITIQUE_FINISH_INSTRUCTION, MAP_REFINEMENT_CRITERIA, MAP_REFINEMENT_FINISH_INSTRUCTION, ) from palimpzest.prompts.filter_prompts import ( FILTER_BASE_SYSTEM_PROMPT, FILTER_BASE_USER_PROMPT, FILTER_NO_REASONING_BASE_SYSTEM_PROMPT, FILTER_NO_REASONING_BASE_USER_PROMPT, ) from palimpzest.prompts.join_prompts import ( JOIN_BASE_SYSTEM_PROMPT, JOIN_BASE_USER_PROMPT, JOIN_NO_REASONING_BASE_SYSTEM_PROMPT, JOIN_NO_REASONING_BASE_USER_PROMPT, ) from palimpzest.prompts.moa_aggregator_prompts import ( FILTER_MOA_AGG_BASE_SYSTEM_PROMPT, FILTER_MOA_AGG_BASE_USER_PROMPT, MAP_MOA_AGG_BASE_SYSTEM_PROMPT, MAP_MOA_AGG_BASE_USER_PROMPT, ) from palimpzest.prompts.moa_proposer_prompts import ( FILTER_MOA_PROPOSER_BASE_SYSTEM_PROMPT, FILTER_MOA_PROPOSER_BASE_USER_PROMPT, MAP_MOA_PROPOSER_BASE_SYSTEM_PROMPT, MAP_MOA_PROPOSER_BASE_USER_PROMPT, ) from palimpzest.prompts.split_merge_prompts import ( FILTER_SPLIT_MERGER_BASE_SYSTEM_PROMPT, FILTER_SPLIT_MERGER_BASE_USER_PROMPT, MAP_SPLIT_MERGER_BASE_SYSTEM_PROMPT, MAP_SPLIT_MERGER_BASE_USER_PROMPT, ) from palimpzest.prompts.split_proposer_prompts import ( FILTER_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT, FILTER_SPLIT_PROPOSER_BASE_USER_PROMPT, MAP_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT, MAP_SPLIT_PROPOSER_BASE_USER_PROMPT, ) from palimpzest.prompts.utils import ( AGG_AUDIO_DISCLAIMER, AGG_EXAMPLE_ANSWER, AGG_EXAMPLE_OUTPUT_FIELDS, AGG_EXAMPLE_REASONING, AGG_IMAGE_DISCLAIMER, AGG_JOB_INSTRUCTION, AUDIO_DISCLAIMER, AUDIO_EXAMPLE_ANSWER, AUDIO_EXAMPLE_CONTEXT, AUDIO_EXAMPLE_INPUT_FIELDS, AUDIO_EXAMPLE_OUTPUT_FIELDS, AUDIO_EXAMPLE_REASONING, AUDIO_SENTENCE_EXAMPLE_ANSWER, DESC_SECTION, EXAMPLE_AGG_INSTRUCTION, EXAMPLE_FILTER_CONDITION, EXAMPLE_JOIN_CONDITION, FILTER_EXAMPLE_REASONING, FILTER_JOB_INSTRUCTION, IMAGE_DISCLAIMER, IMAGE_EXAMPLE_ANSWER, IMAGE_EXAMPLE_CONTEXT, IMAGE_EXAMPLE_INPUT_FIELDS, IMAGE_EXAMPLE_OUTPUT_FIELDS, IMAGE_EXAMPLE_REASONING, IMAGE_SENTENCE_EXAMPLE_ANSWER, JOIN_EXAMPLE_REASONING, JOIN_JOB_INSTRUCTION, MAP_JOB_INSTRUCTION, ONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION, ONE_TO_ONE_OUTPUT_FORMAT_INSTRUCTION, PROPOSER_JOB_INSTRUCTION, RIGHT_AUDIO_DISCLAIMER, RIGHT_AUDIO_EXAMPLE_CONTEXT, RIGHT_AUDIO_EXAMPLE_INPUT_FIELDS, RIGHT_IMAGE_DISCLAIMER, RIGHT_IMAGE_EXAMPLE_CONTEXT, RIGHT_IMAGE_EXAMPLE_INPUT_FIELDS, RIGHT_TEXT_EXAMPLE_CONTEXT, RIGHT_TEXT_EXAMPLE_INPUT_FIELDS, SECOND_AUDIO_EXAMPLE_CONTEXT, SECOND_IMAGE_EXAMPLE_CONTEXT, SECOND_TEXT_EXAMPLE_CONTEXT, TEXT_EXAMPLE_ANSWER, TEXT_EXAMPLE_CONTEXT, TEXT_EXAMPLE_INPUT_FIELDS, TEXT_EXAMPLE_OUTPUT_FIELDS, TEXT_EXAMPLE_REASONING, TEXT_SENTENCE_EXAMPLE_ANSWER, THIRD_AUDIO_EXAMPLE_CONTEXT, THIRD_IMAGE_EXAMPLE_CONTEXT, THIRD_TEXT_EXAMPLE_CONTEXT, ) def _detect_image_media_type(filepath: str | None = None, base64_data: str | None = None) -> str: """Detect image media type from file extension or base64 magic bytes.""" if filepath: ext = os.path.splitext(filepath)[1].lower() ext_map = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".webp": "image/webp"} if ext in ext_map: return ext_map[ext] if base64_data: try: header = base64.b64decode(base64_data[:32]) if header[:8] == b"\x89PNG\r\n\x1a\n": return "image/png" if header[:3] == b"\xff\xd8\xff": return "image/jpeg" if header[:6] in (b"GIF87a", b"GIF89a"): return "image/gif" if header[:4] == b"RIFF" and header[8:12] == b"WEBP": return "image/webp" except Exception: pass return "image/jpeg" class PromptFactory: """Factory class for generating prompts for the Generator given the input(s).""" BASE_SYSTEM_PROMPT_MAP = { # agg user prompts PromptStrategy.AGG: AGG_BASE_SYSTEM_PROMPT, PromptStrategy.AGG_NO_REASONING: AGG_NO_REASONING_BASE_SYSTEM_PROMPT, # filter system prompts PromptStrategy.FILTER: FILTER_BASE_SYSTEM_PROMPT, PromptStrategy.FILTER_NO_REASONING: FILTER_NO_REASONING_BASE_SYSTEM_PROMPT, PromptStrategy.FILTER_CRITIC: None, PromptStrategy.FILTER_REFINE: None, PromptStrategy.FILTER_MOA_PROPOSER: FILTER_MOA_PROPOSER_BASE_SYSTEM_PROMPT, PromptStrategy.FILTER_MOA_AGG: FILTER_MOA_AGG_BASE_SYSTEM_PROMPT, PromptStrategy.FILTER_SPLIT_PROPOSER: FILTER_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT, PromptStrategy.FILTER_SPLIT_MERGER: FILTER_SPLIT_MERGER_BASE_SYSTEM_PROMPT, # join system prompts PromptStrategy.JOIN: JOIN_BASE_SYSTEM_PROMPT, PromptStrategy.JOIN_NO_REASONING: JOIN_NO_REASONING_BASE_SYSTEM_PROMPT, # map system prompts PromptStrategy.MAP: MAP_BASE_SYSTEM_PROMPT, PromptStrategy.MAP_NO_REASONING: MAP_NO_REASONING_BASE_SYSTEM_PROMPT, PromptStrategy.MAP_CRITIC: None, PromptStrategy.MAP_REFINE: None, PromptStrategy.MAP_MOA_PROPOSER: MAP_MOA_PROPOSER_BASE_SYSTEM_PROMPT, PromptStrategy.MAP_MOA_AGG: MAP_MOA_AGG_BASE_SYSTEM_PROMPT, PromptStrategy.MAP_SPLIT_PROPOSER: MAP_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT, PromptStrategy.MAP_SPLIT_MERGER: MAP_SPLIT_MERGER_BASE_SYSTEM_PROMPT, } BASE_USER_PROMPT_MAP = { # agg user prompts PromptStrategy.AGG: AGG_BASE_USER_PROMPT, PromptStrategy.AGG_NO_REASONING: AGG_NO_REASONING_BASE_USER_PROMPT, # filter user prompts PromptStrategy.FILTER: FILTER_BASE_USER_PROMPT, PromptStrategy.FILTER_NO_REASONING: FILTER_NO_REASONING_BASE_USER_PROMPT, PromptStrategy.FILTER_CRITIC: BASE_CRITIQUE_PROMPT, PromptStrategy.FILTER_REFINE: BASE_REFINEMENT_PROMPT, PromptStrategy.FILTER_MOA_PROPOSER: FILTER_MOA_PROPOSER_BASE_USER_PROMPT, PromptStrategy.FILTER_MOA_AGG: FILTER_MOA_AGG_BASE_USER_PROMPT, PromptStrategy.FILTER_SPLIT_PROPOSER: FILTER_SPLIT_PROPOSER_BASE_USER_PROMPT, PromptStrategy.FILTER_SPLIT_MERGER: FILTER_SPLIT_MERGER_BASE_USER_PROMPT, # join user prompts PromptStrategy.JOIN: JOIN_BASE_USER_PROMPT, PromptStrategy.JOIN_NO_REASONING: JOIN_NO_REASONING_BASE_USER_PROMPT, # map user prompts PromptStrategy.MAP: MAP_BASE_USER_PROMPT, PromptStrategy.MAP_NO_REASONING: MAP_NO_REASONING_BASE_USER_PROMPT, PromptStrategy.MAP_CRITIC: BASE_CRITIQUE_PROMPT, PromptStrategy.MAP_REFINE: BASE_REFINEMENT_PROMPT, PromptStrategy.MAP_MOA_PROPOSER: MAP_MOA_PROPOSER_BASE_USER_PROMPT, PromptStrategy.MAP_MOA_AGG: MAP_MOA_AGG_BASE_USER_PROMPT, PromptStrategy.MAP_SPLIT_PROPOSER: MAP_SPLIT_PROPOSER_BASE_USER_PROMPT, PromptStrategy.MAP_SPLIT_MERGER: MAP_SPLIT_MERGER_BASE_USER_PROMPT, } def __init__(self, prompt_strategy: PromptStrategy, model: Model, cardinality: Cardinality, desc: str | None = None) -> None: self.prompt_strategy = prompt_strategy self.model = model self.cardinality = cardinality self.desc = desc def _get_context(self, candidate: DataRecord | list[DataRecord], input_fields: list[str]) -> str: """ Returns the context for the prompt. Args: candidate (DataRecord): The input record. input_fields (list[str]): The input fields. Returns: str: The context. """ # TODO: remove mask_filepaths=True after SemBench evaluation # get context from input record (project_cols will be None if not provided in kwargs) if isinstance(candidate, list): context: list[dict] = [record.to_dict(include_bytes=False, project_cols=input_fields, mask_filepaths=True) for record in candidate] else: context: dict = candidate.to_dict(include_bytes=False, project_cols=input_fields, mask_filepaths=True) # TODO: MOVE THIS LOGIC INTO A CHUNKING / CONTEXT MANAGEMENT CLASS # - this class should be able to: # - handle the context length of different models (i.e. self.model should be an input) # - handle images # - handle the issue with `original_messages` (ask Matt if this is not clear) # TODO: this does not work for image prompts # TODO: this ignores the size of the `orignal_messages` in critique and refine prompts # NOTE: llama models are disallowed for aggregation so we can assume context is a dict here # cut down on context based on window length if self.model.is_llama_model(): assert isinstance(context, dict), "Llama models are not allowed for aggregation operations." total_context_len = len(json.dumps(context, indent=2)) # sort fields by length and progressively strip from the longest field until it is short enough; # NOTE: LLAMA_CONTEXT_TOKENS_LIMIT is a rough estimate which leaves room for the rest of the prompt text while total_context_len * TOKENS_PER_CHARACTER > LLAMA_CONTEXT_TOKENS_LIMIT: # sort fields by length field_lengths = [(field, len(value) if value is not None else 0) for field, value in context.items()] sorted_fields = sorted(field_lengths, key=lambda item: item[1], reverse=True) # get field with longest context longest_field_name, longest_field_length = sorted_fields[0] # trim the field context_factor = LLAMA_CONTEXT_TOKENS_LIMIT / (total_context_len * TOKENS_PER_CHARACTER) keep_frac_idx = int(longest_field_length * context_factor) context[longest_field_name] = context[longest_field_name][:keep_frac_idx] # update total context length total_context_len = len(json.dumps(context, indent=2)) return json.dumps(context, indent=2) def _get_input_fields(self, candidate: DataRecord, **kwargs) -> list[str]: """ The list of input fields to be templated into the prompt(s). If the user provides a list of "project_cols" in kwargs, then this list will be returned. Otherwise, this function returns the list of all field names in the candidate record. Args: candidate (DataRecord): The input record. kwargs: The keyword arguments provided by the user. Returns: list[str]: The list of input field names. """ # NOTE: joins will include left and right input fields in project_cols, so we have to check # if the field is in the candidate record input_fields = kwargs.get("project_cols", candidate.get_field_names()) input_fields = [field for field in input_fields if field in candidate.get_field_names()] return input_fields def _get_input_modalities(self, candidate: DataRecord, input_fields: list[str]) -> set[Modality]: """ The list of input modalities for the given input fields. Args: candidate (DataRecord): The input record. input_fields (list[str]): The input fields. Returns: set[Modality]: The list of input modalities. """ input_modalities = [] for field_name in input_fields: field_type = candidate.get_field_type(field_name) if field_type.annotation in IMAGE_FIELD_TYPES: input_modalities.append(Modality.IMAGE) elif field_type.annotation in AUDIO_FIELD_TYPES: input_modalities.append(Modality.AUDIO) else: input_modalities.append(Modality.TEXT) return set(input_modalities) def _get_modalities_str(self, input_modalities: set[Modality]) -> str: """ Returns a format string to reflect the input modalities. Args: input_modalities (set[Modality]): The input modalities. Returns: str: The string to reflect the input modalities. """ if input_modalities == {Modality.TEXT}: return "text" elif input_modalities == {Modality.IMAGE}: return "image(s)" elif input_modalities == {Modality.AUDIO}: return "audio" elif input_modalities == {Modality.TEXT, Modality.IMAGE}: return "text and/or image(s)" elif input_modalities == {Modality.TEXT, Modality.AUDIO}: return "text and/or audio" elif input_modalities == {Modality.IMAGE, Modality.AUDIO}: return "image(s) and/or audio" elif input_modalities == {Modality.TEXT, Modality.IMAGE, Modality.AUDIO}: return "text, image(s), and/or audio" def _get_input_fields_desc(self, candidate: DataRecord, input_fields: list[str]) -> str: """ Returns a multi-line description of each input field for the prompt. Args: input_fields (list[str]): The input fields. Returns: str: The input fields description. """ input_fields_desc = "" for field_name in input_fields: input_fields_desc += f"- {field_name}: {candidate.get_field_type(field_name).description}\n" return input_fields_desc[:-1] def _get_output_fields_desc(self, output_fields: list[str], **kwargs) -> str: """ Returns a multi-line description of each output field for the prompt. Args: output_fields (list[str]): The output fields. kwargs: The keyword arguments provided by the user. Returns: str: The output fields description. """ output_fields_desc = "" output_schema: type[BaseModel] = kwargs.get("output_schema") if self.prompt_strategy.is_map_prompt() or self.prompt_strategy.is_agg_prompt(): assert output_schema is not None, "Output schema must be provided for convert prompts." for field_name in sorted(output_fields): desc = output_schema.model_fields[field_name].description output_fields_desc += f"- {field_name}: {'no description available' if desc is None else desc}\n" # strip the last newline characters from the field descriptions and return return output_fields_desc[:-1] def _get_agg_instruction(self, **kwargs) -> str | None: """ Returns the aggregation instruction for the aggregation operation. Returns: str | None: The aggregation instruction (if applicable). """ agg_instruction = kwargs.get("agg_instruction") if self.prompt_strategy.is_agg_prompt(): assert agg_instruction is not None, "Aggregation instruction must be provided for aggregation operations." return agg_instruction def _get_filter_condition(self, **kwargs) -> str | None: """ Returns the filter condition for the filter operation. Returns: str | None: The filter condition (if applicable). """ filter_condition = kwargs.get("filter_condition") if self.prompt_strategy.is_filter_prompt(): assert filter_condition is not None, "Filter condition must be provided for filter operations." return filter_condition def _get_join_condition(self, **kwargs) -> str | None: """ Returns the join condition for the join operation. Returns: str | None: The join condition (if applicable). """ join_condition = kwargs.get("join_condition") if self.prompt_strategy.is_join_prompt(): assert join_condition is not None, "Join condition must be provided for join operations." return join_condition def _get_original_output(self, **kwargs) -> str | None: """ Returns the original output from a previous model generation for the critique and refinement operations. Args: kwargs: The keyword arguments provided by the user. Returns: str | None: The original output. """ original_output = kwargs.get("original_output") if self.prompt_strategy.is_critic_prompt() or self.prompt_strategy.is_refine_prompt(): assert original_output is not None, ( "Original output must be provided for critique and refinement operations." ) return original_output def _get_critique_output(self, **kwargs) -> str | None: """ Returns the critique output for the refinement operation. Args: kwargs: The keyword arguments provided by the user. Returns: str | None: The critique output. """ critique_output = kwargs.get("critique_output") if self.prompt_strategy.is_refine_prompt(): assert critique_output is not None, "Critique output must be provided for refinement operations." return critique_output def _get_model_responses(self, **kwargs) -> str | None: """ Returns the model responses for the mixture-of-agents aggregation operation. Args: kwargs: The keyword arguments provided by the user. Returns: str | None: The model responses. """ model_responses = None if self.prompt_strategy.is_moa_aggregator_prompt(): model_responses = "" for idx, model_response in enumerate(kwargs.get("model_responses")): model_responses += f"MODEL RESPONSE {idx + 1}: {model_response.rstrip()}\n\n" model_responses = model_responses.rstrip() if model_responses is not None else None return model_responses def _get_chunk_outputs(self, **kwargs) -> str | None: """ Returns the chunk outputs for the split-convert. Args: kwargs: The keyword arguments provided by the user. Returns: str | None: The chunk outputs. """ chunk_outputs = None if self.prompt_strategy.is_split_merger_prompt(): chunk_outputs = "" for idx, chunk_output in enumerate(kwargs.get("chunk_outputs")): chunk_outputs += f"CHUNK OUTPUT {idx + 1}: {chunk_output.rstrip()}\n\n" chunk_outputs = chunk_outputs.rstrip() if chunk_outputs is not None else None return chunk_outputs def _get_output_format_instruction(self) -> str: """ Returns the output format instruction based on the cardinality. Returns: str: The output format instruction. """ return ( ONE_TO_ONE_OUTPUT_FORMAT_INSTRUCTION if self.cardinality == Cardinality.ONE_TO_ONE else ONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION ) def _get_job_instruction(self, input_modalities: set[Modality]) -> str | None: """ Returns the job instruction based on the prompt strategy. Args: input_modalities (set[Modality]): The modalities of the input fields. Returns: str | None: The job instruction. """ # get the job instruction based on the prompt strategy job_instruction = None if self.prompt_strategy.is_moa_proposer_prompt() or self.prompt_strategy.is_split_proposer_prompt(): job_instruction = PROPOSER_JOB_INSTRUCTION elif self.prompt_strategy.is_map_prompt(): job_instruction = MAP_JOB_INSTRUCTION elif self.prompt_strategy.is_filter_prompt(): job_instruction = FILTER_JOB_INSTRUCTION elif self.prompt_strategy.is_join_prompt(): job_instruction = JOIN_JOB_INSTRUCTION elif self.prompt_strategy.is_agg_prompt(): job_instruction = AGG_JOB_INSTRUCTION # format the job instruction based on the input modalities modalities = self._get_modalities_str(input_modalities) if job_instruction is not None: job_instruction = job_instruction.format(modalities=modalities) return job_instruction def _get_desc_section(self) -> str: """ Returns the description section for the prompt. Returns: str: The description section (if applicable). """ desc_section = "" if self.desc is not None: desc_section = DESC_SECTION.format(desc=self.desc) return desc_section def _get_critique_criteria(self) -> str | None: """ Returns the critique criteria for the critique operation. Returns: str | None: The critique criteria (if applicable). """ critique_criteria = None if self.prompt_strategy.is_critic_prompt(): critique_criteria = MAP_CRITIQUE_CRITERIA if self.prompt_strategy.is_map_prompt() else FILTER_CRITIQUE_CRITERIA return critique_criteria def _get_refinement_criteria(self) -> str | None: """ Returns the refinement criteria for the refinement operation. Returns: str | None: The refinement criteria (if applicable). """ refinement_criteria = None if self.prompt_strategy.is_refine_prompt(): refinement_criteria = MAP_REFINEMENT_CRITERIA if self.prompt_strategy.is_map_prompt() else FILTER_REFINEMENT_CRITERIA return refinement_criteria def _get_finish_instruction(self) -> str | None: """ Returns the finish instruction for the critique and refinement operations. Returns: str | None: The finish instruction (if applicable). """ finish_instruction = None if self.prompt_strategy.is_critic_prompt(): finish_instruction = MAP_CRITIQUE_FINISH_INSTRUCTION if self.prompt_strategy.is_map_prompt() else FILTER_CRITIQUE_FINISH_INSTRUCTION elif self.prompt_strategy.is_refine_prompt(): finish_instruction = MAP_REFINEMENT_FINISH_INSTRUCTION if self.prompt_strategy.is_map_prompt() else FILTER_REFINEMENT_FINISH_INSTRUCTION return finish_instruction def _get_example_input_fields(self, input_modalities: set[Modality], right: bool = False) -> str: """ Returns the example input fields for the prompt. Args: input_modalities (set[Modality]): The modalities of the input fields. right (bool): Whether to return the right input fields for the join prompt. Returns: str: The example input fields. """ input_modality_to_example_input_fields = { Modality.TEXT: RIGHT_TEXT_EXAMPLE_INPUT_FIELDS if right else TEXT_EXAMPLE_INPUT_FIELDS, Modality.IMAGE: RIGHT_IMAGE_EXAMPLE_INPUT_FIELDS if right else IMAGE_EXAMPLE_INPUT_FIELDS, Modality.AUDIO: RIGHT_AUDIO_EXAMPLE_INPUT_FIELDS if right else AUDIO_EXAMPLE_INPUT_FIELDS, } example_input_fields = "" for input_modality in input_modalities: example_input_fields += input_modality_to_example_input_fields[input_modality].rstrip() example_input_fields = example_input_fields.lstrip() + "\n" return example_input_fields def _get_example_output_fields(self, input_modalities: set[Modality]) -> str: """ Returns the example output fields for the prompt. Returns: str: The example output fields. """ if self.prompt_strategy.is_agg_prompt(): return AGG_EXAMPLE_OUTPUT_FIELDS input_modality_to_example_output_fields = { Modality.TEXT: TEXT_EXAMPLE_OUTPUT_FIELDS, Modality.IMAGE: IMAGE_EXAMPLE_OUTPUT_FIELDS, Modality.AUDIO: AUDIO_EXAMPLE_OUTPUT_FIELDS, } example_output_fields = "" for input_modality in input_modalities: example_output_fields += input_modality_to_example_output_fields[input_modality].rstrip() example_output_fields = example_output_fields.lstrip() + "\n" return example_output_fields def _get_example_context(self, input_modalities: set[Modality], right: bool = False, second: bool = False, third: bool = False) -> str: """ Returns the example context for the prompt. Returns: str: The example context. """ assert not (second and third), "Cannot have both second and third example contexts." assert not (right and (second or third)), "Right context is only used for joins; second and third contexts only use for aggregations." text_example_context = TEXT_EXAMPLE_CONTEXT image_example_context = IMAGE_EXAMPLE_CONTEXT audio_example_context = AUDIO_EXAMPLE_CONTEXT if second: text_example_context = SECOND_TEXT_EXAMPLE_CONTEXT image_example_context = SECOND_IMAGE_EXAMPLE_CONTEXT audio_example_context = SECOND_AUDIO_EXAMPLE_CONTEXT elif third: text_example_context = THIRD_TEXT_EXAMPLE_CONTEXT image_example_context = THIRD_IMAGE_EXAMPLE_CONTEXT audio_example_context = THIRD_AUDIO_EXAMPLE_CONTEXT input_modality_to_example_context = { Modality.TEXT: RIGHT_TEXT_EXAMPLE_CONTEXT if right else text_example_context, Modality.IMAGE: RIGHT_IMAGE_EXAMPLE_CONTEXT if right else image_example_context, Modality.AUDIO: RIGHT_AUDIO_EXAMPLE_CONTEXT if right else audio_example_context, } example_context = "" for input_modality in input_modalities: example_context += input_modality_to_example_context[input_modality].rstrip() + "," example_context = example_context[:-1] + "\n" return example_context def _get_image_disclaimer(self, input_modalities: set[Modality], right: bool = False, agg: bool = False) -> str: """ Returns the image disclaimer for the prompt. The disclaimer must be an empty string for non-image prompts. Returns: str: The image disclaimer. If this is a text prompt then it is an empty string. """ assert not (right and agg), "Right image disclaimer is only used for joins; agg image disclaimer only used for aggregations." image_disclaimer = AGG_IMAGE_DISCLAIMER if agg else IMAGE_DISCLAIMER image_disclaimer = RIGHT_IMAGE_DISCLAIMER if right else image_disclaimer return image_disclaimer if Modality.IMAGE in input_modalities else "" def _get_audio_disclaimer(self, input_modalities: set[Modality], right: bool = False, agg: bool = False) -> str: """ Returns the audio disclaimer for the prompt. The disclaimer must be an empty string for non-audio prompts. Returns: str: The audio disclaimer. If this is a text prompt then it is an empty string. """ assert not (right and agg), "Right audio disclaimer is only used for joins; agg audio disclaimer only used for aggregations." audio_disclaimer = AGG_AUDIO_DISCLAIMER if agg else AUDIO_DISCLAIMER audio_disclaimer = RIGHT_AUDIO_DISCLAIMER if right else audio_disclaimer return audio_disclaimer if Modality.AUDIO in input_modalities else "" def _get_example_reasoning(self, input_modalities: set[Modality]) -> str: """ Returns the example reasoning for the prompt. Returns: str: The example reasoning. """ if self.prompt_strategy.is_filter_prompt(): return FILTER_EXAMPLE_REASONING elif self.prompt_strategy.is_join_prompt(): return JOIN_EXAMPLE_REASONING elif self.prompt_strategy.is_agg_prompt(): return AGG_EXAMPLE_REASONING input_modality_to_example_reasoning = { Modality.TEXT: TEXT_EXAMPLE_REASONING, Modality.IMAGE: IMAGE_EXAMPLE_REASONING, Modality.AUDIO: AUDIO_EXAMPLE_REASONING, } example_reasoning = "" for input_modality in input_modalities: example_reasoning += input_modality_to_example_reasoning[input_modality] + " " example_reasoning = example_reasoning.rstrip() return example_reasoning def _get_example_answer(self, input_modalities: set[Modality]) -> str: """ Returns the example answer for the prompt. Returns: str: The example answer. """ if self.prompt_strategy.is_agg_prompt(): return AGG_EXAMPLE_ANSWER use_sentence_answers = self.prompt_strategy.is_split_proposer_prompt() or self.prompt_strategy.is_moa_proposer_prompt() input_modality_to_example_answer = { Modality.TEXT: TEXT_SENTENCE_EXAMPLE_ANSWER if use_sentence_answers else TEXT_EXAMPLE_ANSWER, Modality.IMAGE: IMAGE_SENTENCE_EXAMPLE_ANSWER if use_sentence_answers else IMAGE_EXAMPLE_ANSWER, Modality.AUDIO: AUDIO_SENTENCE_EXAMPLE_ANSWER if use_sentence_answers else AUDIO_EXAMPLE_ANSWER, } example_answer = "" for input_modality in input_modalities: example_answer += input_modality_to_example_answer[input_modality].rstrip() if use_sentence_answers: example_answer += " " example_answer = example_answer + "\n" return example_answer def _get_all_format_kwargs( self, candidate: DataRecord | list[DataRecord], input_fields: list[str], input_modalities: set[Modality], output_fields: list[str], right_candidate: DataRecord | None, right_input_fields: list[str], right_input_modalities: set[Modality], **kwargs, ) -> dict: """ Returns a dictionary containing all the format kwargs for templating the prompts. Args: candidate (DataRecord): The input record. input_fields (list[str]): The input fields. output_fields (list[str]): The output fields. kwargs: The keyword arguments provided by the user. Returns: dict: The dictionary containing all the format kwargs. """ # get format kwargs which depend on the input data input_format_kwargs = { "context": self._get_context(candidate, input_fields), "input_fields_desc": self._get_input_fields_desc(candidate[0] if isinstance(candidate, list) else candidate, input_fields), "output_fields_desc": self._get_output_fields_desc(output_fields, **kwargs), "agg_instruction": self._get_agg_instruction(**kwargs), "filter_condition": self._get_filter_condition(**kwargs), "join_condition": self._get_join_condition(**kwargs), "original_output": self._get_original_output(**kwargs), "critique_output": self._get_critique_output(**kwargs), "model_responses": self._get_model_responses(**kwargs), "chunk_outputs": self._get_chunk_outputs(**kwargs), } # if a right candidate is provided, we also get the context and input field descriptions for the right candidate if right_candidate is not None: input_format_kwargs.update({ "right_context": self._get_context(right_candidate, right_input_fields), "right_input_fields_desc": self._get_input_fields_desc(right_candidate, right_input_fields), }) # get format kwargs which depend on the prompt strategy full_input_modalities = input_modalities.union(right_input_modalities) prompt_strategy_format_kwargs = { "output_format_instruction": self._get_output_format_instruction(), "job_instruction": self._get_job_instruction(full_input_modalities), "desc_section": self._get_desc_section(), "critique_criteria": self._get_critique_criteria(), "refinement_criteria": self._get_refinement_criteria(), "finish_instruction": self._get_finish_instruction(), "example_input_fields": self._get_example_input_fields(input_modalities), "right_example_input_fields": self._get_example_input_fields(right_input_modalities, right=True), "example_output_fields": self._get_example_output_fields(input_modalities), "example_context": self._get_example_context(input_modalities), "second_example_context": self._get_example_context(input_modalities, second=True) if self.prompt_strategy.is_agg_prompt() else "", "third_example_context": self._get_example_context(input_modalities, third=True) if self.prompt_strategy.is_agg_prompt() else "", "right_example_context": self._get_example_context(right_input_modalities, right=True), "image_disclaimer": self._get_image_disclaimer(input_modalities, agg=self.prompt_strategy.is_agg_prompt()), "audio_disclaimer": self._get_audio_disclaimer(input_modalities, agg=self.prompt_strategy.is_agg_prompt()), "right_image_disclaimer": self._get_image_disclaimer(right_input_modalities, right=True), "right_audio_disclaimer": self._get_audio_disclaimer(right_input_modalities, right=True), "example_agg_instruction": EXAMPLE_AGG_INSTRUCTION, "example_filter_condition": EXAMPLE_FILTER_CONDITION, "example_join_condition": EXAMPLE_JOIN_CONDITION, "example_reasoning": self._get_example_reasoning(input_modalities), "example_answer": self._get_example_answer(input_modalities), } # return all format kwargs return {**input_format_kwargs, **prompt_strategy_format_kwargs} def _create_audio_messages(self, candidate: DataRecord | list[DataRecord], input_fields: list[str]) -> list[dict]: """ Parses the candidate record(s) and returns the audio messages for the chat payload. Args: candidate (DataRecord | list[DataRecord]): The input record(s). input_fields (list[str]): The list of input fields. Returns: list[dict]: The audio messages for the chat payload. """ # normalize type to be list[DataRecord] if isinstance(candidate, DataRecord): candidate = [candidate] # create a message for each audio recording in an input field with an audio (or list of audio) type audio_content = [] for field_name in input_fields: for dr in candidate: field_value = dr[field_name] field_type = dr.get_field_type(field_name) # audio filepath (or list of audio filepaths) if field_type.annotation in [AudioFilepath, AudioFilepath | None, AudioFilepath | Any] and field_value is not None: with open(field_value, "rb") as f: base64_audio_str = base64.b64encode(f.read()).decode("utf-8") audio_content.append( {"type": "input_audio", "input_audio": {"data": base64_audio_str, "format": "wav"}} ) elif field_type.annotation in [list[AudioFilepath], list[AudioFilepath] | None, list[AudioFilepath] | Any]: for audio_filepath in field_value: if audio_filepath is None: continue with open(audio_filepath, "rb") as f: base64_audio_str = base64.b64encode(f.read()).decode("utf-8") audio_content.append( {"type": "input_audio", "input_audio": {"data": base64_audio_str, "format": "wav"}} ) # pre-encoded images (or list of pre-encoded images) elif field_type.annotation in [AudioBase64, AudioBase64 | None, AudioBase64 | Any] and field_value is not None: audio_content.append( {"type": "input_audio", "input_audio": {"data": field_value, "format": "wav"}} ) elif field_type.annotation in [list[AudioBase64], list[AudioBase64] | None, list[AudioBase64] | Any]: for base64_audio in field_value: if base64_audio is None: continue audio_content.append( {"type": "input_audio", "input_audio": {"data": base64_audio, "format": "wav"}} ) return [{"role": "user", "type": "input_audio", "content": audio_content}] if len(audio_content) > 0 else [] def _create_image_messages(self, candidate: DataRecord | list[DataRecord], input_fields: list[str]) -> list[dict]: """ Parses the candidate record(s) and returns the image messages for the chat payload. Args: candidate (DataRecord | list[DataRecord]): The input record(s). input_fields (list[str]): The list of input fields. Returns: list[dict]: The image messages for the chat payload. """ # normalize type to be list[DataRecord] if isinstance(candidate, DataRecord): candidate = [candidate] # create a message for each image in an input field with an image (or list of image) type image_content = [] for field_name in input_fields: for dr in candidate: field_value = dr[field_name] field_type = dr.get_field_type(field_name) # image filepath (or list of image filepaths) if field_type.annotation in [ImageFilepath, ImageFilepath | None, ImageFilepath | Any] and field_value is not None: with open(field_value, "rb") as f: base64_image_str = base64.b64encode(f.read()).decode("utf-8") media_type = _detect_image_media_type(filepath=field_value, base64_data=base64_image_str) image_content.append( {"type": "image_url", "image_url": {"url": f"data:{media_type};base64,{base64_image_str}"}} ) elif field_type.annotation in [list[ImageFilepath], list[ImageFilepath] | None, list[ImageFilepath] | Any]: for image_filepath in field_value: if image_filepath is None: continue with open(image_filepath, "rb") as f: base64_image_str = base64.b64encode(f.read()).decode("utf-8") media_type = _detect_image_media_type(filepath=image_filepath, base64_data=base64_image_str) image_content.append( {"type": "image_url", "image_url": {"url": f"data:{media_type};base64,{base64_image_str}"}} ) # image url (or list of image urls) elif field_type.annotation in [ImageURL, ImageURL | None, ImageURL | Any] and field_value is not None: image_content.append({"type": "image_url", "image_url": {"url": field_value}}) elif field_type.annotation in [list[ImageURL], list[ImageURL] | None, list[ImageURL] | Any]: for image_url in field_value: if image_url is None: continue image_content.append({"type": "image_url", "image_url": {"url": image_url}}) # pre-encoded images (or list of pre-encoded images) elif field_type.annotation in [ImageBase64, ImageBase64 | None, ImageBase64 | Any] and field_value is not None: media_type = _detect_image_media_type(base64_data=field_value) image_content.append( {"type": "image_url", "image_url": {"url": f"data:{media_type};base64,{field_value}"}} ) elif field_type.annotation in [list[ImageBase64], list[ImageBase64] | None, list[ImageBase64] | Any]: for base64_image in field_value: if base64_image is None: continue media_type = _detect_image_media_type(base64_data=base64_image) image_content.append( {"type": "image_url", "image_url": {"url": f"data:{media_type};base64,{base64_image}"}} ) return [{"role": "user", "type": "image", "content": image_content}] if len(image_content) > 0 else [] def _get_system_prompt(self, **format_kwargs) -> str | None: """ Returns the fully templated system prompt for the given prompt strategy. Returns None if the prompt strategy does not use a system prompt. Returns: str | None: The fully templated system prompt (or None if the prompt strategy does not use a system prompt). """ base_prompt: str = self.BASE_SYSTEM_PROMPT_MAP.get(self.prompt_strategy) # for critic and refine prompt strategies, we do not use a base prompt if base_prompt is None: return base_prompt return base_prompt.format(**format_kwargs) def _get_user_messages(self, candidate: DataRecord | list[DataRecord], input_fields: list[str], right_candidate: DataRecord | None, right_input_fields: list[str], **kwargs) -> str: """ Returns a list of messages for the chat payload based on the prompt strategy. Args: candidate (DataRecord | list[DataRecord]): The input record(s). input_fields (list[str]): The input fields. output_fields (list[str]): The output fields. kwargs: The formatting kwargs and some keyword arguments provided by the user. Returns: Tuple[str, str | None]: The fully templated start and end of the user prompt. The second element will be None for text prompts. """ # get the base prompt template base_prompt = self.BASE_USER_PROMPT_MAP.get(self.prompt_strategy) # get any image messages for the chat payload (will be an empty list if no image fields exist) image_messages = self._create_image_messages(candidate, input_fields) # get any audio messages for the chat payload (will be an empty list if no audio fields exist) audio_messages = self._create_audio_messages(candidate, input_fields) # get any right image / audio messages for the chat payload (will be an empty list if image / audio not present) right_image_messages, right_audio_messages = [], [] if self.prompt_strategy.is_join_prompt(): assert right_candidate is not None, "Right candidate must be provided for join prompts." right_image_messages = self._create_image_messages(right_candidate, right_input_fields) right_audio_messages = self._create_audio_messages(right_candidate, right_input_fields) # get any original messages for critique and refinement operations original_messages = kwargs.get("original_messages") if self.prompt_strategy.is_critic_prompt() or self.prompt_strategy.is_refine_prompt(): assert original_messages is not None, ( "Original messages must be provided for critique and refinement operations." ) # combine image and audio messages image_audio_messages = image_messages + audio_messages right_image_audio_messages = right_image_messages + right_audio_messages has_image_audio = len(image_audio_messages) > 0 has_right_image_audio = len(right_image_audio_messages) > 0 # construct the user messages based on the prompt strategy user_messages = [] if self.prompt_strategy.is_critic_prompt() or self.prompt_strategy.is_refine_prompt(): # NOTE: if this critic / refinement prompt is processing images / audio, those images / audio # will be part of the `original_messages` and will show up in the final chat payload base_prompt_start, base_prompt_end = base_prompt.split("<>\n") user_messages.append({"role": "user", "type": "text", "content": base_prompt_start.format(**kwargs)}) user_messages.extend(original_messages) user_messages.append({"role": "user", "type": "text", "content": base_prompt_end.format(**kwargs)}) # handle joins with left and right images / audio elif self.prompt_strategy.is_join_prompt() and has_image_audio and has_right_image_audio: base_prompt_start, base_prompt_rest = base_prompt.split("<>") base_prompt_mid, base_prompt_end = base_prompt_rest.split("<>") user_messages.append({"role": "user", "type": "text", "content": base_prompt_start.format(**kwargs)}) user_messages.extend(image_audio_messages) user_messages.append({"role": "user", "type": "text", "content": base_prompt_mid.format(**kwargs)}) user_messages.extend(right_image_audio_messages) user_messages.append({"role": "user", "type": "text", "content": base_prompt_end.format(**kwargs)}) # handle joins with only left images / audio elif self.prompt_strategy.is_join_prompt() and has_image_audio and not has_right_image_audio: base_prompt = base_prompt.replace("<>", "") base_prompt_start, base_prompt_end = base_prompt.split("<>") user_messages.append({"role": "user", "type": "text", "content": base_prompt_start.format(**kwargs)}) user_messages.extend(image_audio_messages) user_messages.append({"role": "user", "type": "text", "content": base_prompt_end.format(**kwargs)}) # handle joins with only right images / audio elif self.prompt_strategy.is_join_prompt() and not has_image_audio and has_right_image_audio: base_prompt = base_prompt.replace("<>", "") base_prompt_start, base_prompt_end = base_prompt.split("<>") user_messages.append({"role": "user", "type": "text", "content": base_prompt_start.format(**kwargs)}) user_messages.extend(right_image_audio_messages) user_messages.append({"role": "user", "type": "text", "content": base_prompt_end.format(**kwargs)}) # handle non-joins with images / audio elif not self.prompt_strategy.is_join_prompt() and has_image_audio and not self.prompt_strategy.is_moa_aggregator_prompt(): base_prompt_start, base_prompt_end = base_prompt.split("<>") user_messages.append({"role": "user", "type": "text", "content": base_prompt_start.format(**kwargs)}) user_messages.extend(image_audio_messages) user_messages.append({"role": "user", "type": "text", "content": base_prompt_end.format(**kwargs)}) # handle prompts w/no images or audio else: base_prompt = base_prompt.replace("<>", "") base_prompt = base_prompt.replace("<>", "") user_messages.append({"role": "user", "type": "text", "content": base_prompt.format(**kwargs)}) return user_messages def create_messages(self, candidate: DataRecord | list[DataRecord], output_fields: list[str], right_candidate: DataRecord | None = None, **kwargs) -> list[dict]: """ Creates the messages for the chat payload based on the prompt strategy. Each message will be a dictionary with the following format: { "role": "user" | "system", "type": "text" | "image", "content": str } Args: candidate (DataRecord | list[DataRecord]): The input record(s). output_fields (list[str]): The output fields. right_candidate (DataRecord | None): The other join input record (only provided for joins). kwargs: The keyword arguments provided by the user. Returns: list[dict]: The messages for the chat payload. """ # compute the set of input fields input_fields = self._get_input_fields(candidate[0] if isinstance(candidate, list) else candidate, **kwargs) right_input_fields = [] if right_candidate is None else self._get_input_fields(right_candidate, **kwargs) # use input fields to determine the left / right input modalities input_modalities = self._get_input_modalities(candidate[0] if isinstance(candidate, list) else candidate, input_fields) right_input_modalities = set() if right_candidate is None else self._get_input_modalities(right_candidate, right_input_fields) # initialize messages messages = [] # compute the full dictionary of format kwargs and add to kwargs format_kwargs = self._get_all_format_kwargs(candidate, input_fields, input_modalities, output_fields, right_candidate, right_input_fields, right_input_modalities, **kwargs) kwargs = {**kwargs, **format_kwargs} # generate system message (if applicable) system_prompt = self._get_system_prompt(**kwargs) if system_prompt is not None: messages.append({"role": "system", "type": "text", "content": system_prompt}) # generate user messages and add to messages user_messages = self._get_user_messages(candidate, input_fields, right_candidate, right_input_fields, **kwargs) messages.extend(user_messages) return messages ================================================ FILE: src/palimpzest/prompts/prompt_manager.py ================================================ """ Prompt caching utility for different LLM providers. This module provides provider-specific prompt caching configurations: - OpenAI: Automatic prefix caching with prompt_cache_key for sticky routing - Gemini (Google AI Studio / Vertex AI): Implicit caching (automatic prefix matching) - Anthropic: Explicit cache_control with ephemeral type on system and user message content """ import copy import uuid from typing import Any from palimpzest.constants import Model class PromptManager: """ Manages prompt caching configurations and message transformations for LLM providers. This class handles: 1. Session-level state (e.g., OpenAI cache keys). 2. Provider-specific request arguments (headers, extra_body). 3. Transformation of messages for providers requiring explicit markers (Anthropic). 4. Normalization of usage statistics. """ CACHE_BOUNDARY_MARKER = "<>" def __init__(self, model: Model): self.model = model # Instance-level state ensures thread safety if we use one manager per plan/execution self.openai_cache_key = f"pz-cache-{uuid.uuid4().hex[:12]}" if (self.model.is_provider_openai() or self.model.is_provider_azure()) else None def get_cache_kwargs(self) -> dict[str, Any]: """ Get provider-specific cache configuration kwargs for litellm.completion(). Returns: A dictionary of kwargs to pass to litellm.completion() for enabling caching """ if not self.model.supports_prompt_caching(): return {} # OpenAI and Azure OpenAI: https://platform.openai.com/docs/guides/prompt-caching # Use prompt_cache_key for sticky routing to the same cache shard if self.model.is_provider_openai() or self.model.is_provider_azure(): return {"extra_body": {"prompt_cache_key": self.openai_cache_key}} else: return {} def inject_cache_isolation_id(self, messages: list[dict], session_id: str) -> list[dict]: """ Inject a cache isolation ID into messages for testing cache behavior per-modality. This must happen BEFORE update_messages_for_caching so the ID becomes part of cached content. """ for msg in messages: role = msg.get("role") content = msg.get("content") if role == "system" and isinstance(content, str) or \ role == "user" and self.model.is_provider_anthropic() and msg.get("type") == "text" and isinstance(content, str): msg["content"] = f"[{session_id}] " + content return messages def update_messages_for_caching(self, messages: list[dict]) -> list[dict]: """ Transform messages to conform to provider-specific caching requirements. - Anthropic: Adds explicit cache_control markers. - Others: Removes the generic cache boundary markers. Returns: The transformed messages list. """ if not self.model.supports_prompt_caching(): return messages # Anthropic: Explicit cache_control with ephemeral type # https://platform.claude.com/docs/en/build-with-claude/prompt-caching if self.model.is_provider_anthropic(): return self._transform_messages_for_anthropic(messages) # implicit caching for Gemini/OpenAI/Azure models that currently support caching # OpenAI: https://platform.openai.com/docs/guides/prompt-caching # Gemini: https://ai.google.dev/gemini-api/docs/caching elif (self.model.is_provider_openai() or self.model.is_provider_azure() or self.model.is_provider_google_ai_studio() or self.model.is_provider_vertex_ai()): return self._remove_cache_boundary_markers(messages) return messages def extract_usage_stats(self, usage: dict, is_audio_op: bool) -> dict[str, int]: """ Normalize cache statistics from provider-specific response formats. """ stats = { "input_text_tokens": 0, "input_image_tokens": 0, # forward looking "input_audio_tokens": 0, "cache_creation_tokens": 0, "cache_read_tokens": 0 } details = usage.get("prompt_tokens_details") or {} if self.model.is_provider_openai() or self.model.is_provider_azure(): # only realtime audio models do, but they are not supported by PZ if self.model.supports_prompt_caching() and not is_audio_op: stats["cache_read_tokens"] = details.get("cached_tokens") or 0 stats["input_text_tokens"] = (usage.get("prompt_tokens") or 0) - stats["cache_read_tokens"] # audio models don't support caching for now elif is_audio_op: stats["input_text_tokens"] = details.get("text_tokens") or 0 stats["input_audio_tokens"] = details.get("audio_tokens") or 0 else: stats["input_text_tokens"] = usage.get("prompt_tokens") or 0 # Moved to Gemini client class, now usage stats are extracted directly in GeminiClient # elif self.model.is_provider_vertex_ai(): # stats["cache_read_tokens"] = usage.get("cache_read_input_tokens") or 0 # if stats["cache_read_tokens"] == 0: # stats["cache_read_tokens"] = details.get("cached_tokens") or 0 # stats["input_text_tokens"] = details.get("text_tokens") or 0 # stats["input_audio_tokens"] = details.get("audio_tokens") or 0 # stats["input_image_tokens"] = details.get("image_tokens") or 0 elif self.model.is_provider_anthropic(): stats["cache_creation_tokens"] = usage.get("cache_creation_input_tokens") or 0 stats["cache_read_tokens"] = usage.get("cache_read_input_tokens") or 0 stats["input_text_tokens"] = max(0, (usage.get("prompt_tokens") or 0) - stats["cache_read_tokens"] - stats["cache_creation_tokens"]) elif self.model.is_vllm_model(): # vLLM does not seem to provide cache statistics through litellm, so we currently have no way # to extract cache read/creation tokens for vLLM models. pass # all other models (assume caching not supported) else: if is_audio_op: stats["input_text_tokens"] = details.get("text_tokens") or 0 stats["input_audio_tokens"] = details.get("audio_tokens") or 0 else: stats["input_text_tokens"] = usage.get("prompt_tokens") or 0 return stats def _remove_cache_boundary_markers(self, messages: list[dict]) -> list[dict]: """ Remove <> markers from user messages. For providers with automatic (implicit) caching (OpenAI, Gemini), we don't need explicit cache markers. This function cleans up the markers from prompts. Args: messages: The list of messages to transform. Returns: A new list of messages with cache boundary markers removed. """ result = [] for message in messages: new_message = message.copy() if new_message.get("role") == "user": content = new_message.get("content", "") if isinstance(content, str) and self.CACHE_BOUNDARY_MARKER in content: new_message["content"] = content.replace(self.CACHE_BOUNDARY_MARKER, "") result.append(new_message) return result def _transform_messages_for_anthropic(self, messages: list[dict]) -> list[dict]: """ Add cache_control markers to system messages and user prompt prefixes for Anthropic models. This transforms messages to: 1. Add cache_control to system message content blocks 2. Convert user messages with <> marker into a single message with multiple content blocks: a. Static prefix block (with cache_control) - cacheable across records b. Dynamic content block (without cache_control) - changes per record Args: messages: The list of messages to transform. Returns: A new list of messages with cache_control markers added. """ result = [] for message in messages: new_message = copy.deepcopy(message) role = new_message.get("role") content = new_message.get("content", "") # 1. Handle System Messages if role == "system": if isinstance(content, str) and content: new_message["content"] = [{ "type": "text", "text": content, "cache_control": {"type": "ephemeral"} }] elif isinstance(content, list) and content: # Apply to last block if it's text last_block = new_message["content"][-1] if isinstance(last_block, dict) and last_block.get("type") == "text": last_block["cache_control"] = {"type": "ephemeral"} # 2. Handle User Messages (The Split Logic) elif role == "user" and isinstance(content, str) and self.CACHE_BOUNDARY_MARKER in content: static, dynamic = content.split(self.CACHE_BOUNDARY_MARKER, 1) new_blocks = [] if static.strip(): new_blocks.append({ "type": "text", "text": static, "cache_control": {"type": "ephemeral"} }) if dynamic.strip(): new_blocks.append({"type": "text", "text": dynamic}) if new_blocks: new_message["content"] = new_blocks else: new_message["content"] = "" result.append(new_message) return result ================================================ FILE: src/palimpzest/prompts/split_merge_prompts.py ================================================ """This file contains prompts for SplitConvert aggregator operations.""" ### SYSTEM PROMPTS ### MAP_SPLIT_MERGER_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to generate a JSON object. You will be presented with one or more outputs produced by a set of models operating on chunks of an input. Your task is to synthesize these responses into a single, high-quality JSON object which fills in the output fields with the correct values. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased, incorrect, or contain duplicates. You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the model responses. {output_format_instruction} Finish your response with a newline character followed by --- An example is shown below: --- CHUNK 1 OUTPUT: the text mentions the scientists "Augusta Ada King, Countess of Lovelace" and "Charles Babbage". It states that King was an English mathematician who worked on Babbage's Analytical Engine. CHUNK 2 OUTPUT: the text passage mentions the scientist "Charles Babbage", who was a mathematician. Therefore, the name output should be ["Charles Babbage"] and the field_of_study output should be ["Mathematician"]. INPUT FIELDS: - text: a text passage describing scientists OUTPUT FIELDS: - name: the list of names for each scientist mentioned in the text - field_of_study: a list with the field of study for each scientist Let's think step-by-step in order to answer the question. REASONING: Looking at both chunk outputs, they specify that the scientists' formal names are "Augusta Ada King" and "Charles Babbage". Chunk Output 2 indicates that Charles Babbage was a Mathematician and Chunk Output 1 says that Augusta Ada King was an English mathematician. Therefore, the name output should be ["Augusta Ada King", "Charles Babbage"] and the field_of_study output should be ["Mathematician", "Mathematician"]. ANSWER: {{ "name": ["Augusta Ada King", "Charles Babbage"], "field_of_study": ["Mathematician", "Mathematician"] }} --- """ FILTER_SPLIT_MERGER_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to answer a TRUE/FALSE question. You will be presented with one or more outputs produced by a set of models operating on chunks of an input. Your task is to synthesize these responses into a single TRUE/FALSE answer. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased, incorrect, or contain duplicates. You will be provided with a description of each input field and the filter condition. Remember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by --- An example is shown below: --- CHUNK 1 OUTPUT: The context describes Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace, who is widely recognized as a foundational figure in computer science. Therefore, the answer is TRUE. CHUNK 2 OUTPUT: Based on the context provided, Ada Lovelace is indeed a foundational computer scientist, therefore the answer is TRUE. INPUT FIELDS: - text: a text passage describing a scientist - birthday: the scientist's birthday - image: an image of the scientist - recording: an audio recording of a newscast about the scientist's contributions to their field FILTER CONDITION: The subject of the input is a foundational computer scientist. Let's think step-by-step in order to answer the question. REASONING: Looking at both chunk outputs, they agree that the subject is a foundational computer scientist. Both outputs provide consistent evidence supporting this conclusion. ANSWER: TRUE --- """ ### USER / INSTANCE-SPECIFIC PROMPTS ### MAP_SPLIT_MERGER_BASE_USER_PROMPT = """You are a helpful assistant whose job is to generate a JSON object. You will be presented with one or more outputs produced by a set of models. Your task is to synthesize these responses into a single, high-quality JSON object which fills in the output fields with the correct values. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased, incorrect, or contain duplicates. You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the model responses. {output_format_instruction} Finish your response with a newline character followed by --- --- INPUT FIELDS: {input_fields_desc} OUTPUT FIELDS: {output_fields_desc} <>{chunk_outputs} Let's think step-by-step in order to answer the question. REASONING: """ FILTER_SPLIT_MERGER_BASE_USER_PROMPT = """You are a helpful assistant whose job is to answer a TRUE/FALSE question. You will be presented with one or more outputs produced by a set of models operating on chunks of an input. Your task is to synthesize these responses into a single TRUE/FALSE answer. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased, incorrect, or contain duplicates. You will be provided with a description of each input field and the filter condition. Remember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by --- --- INPUT FIELDS: {input_fields_desc} FILTER CONDITION: {filter_condition} <>{chunk_outputs} Let's think step-by-step in order to answer the question. REASONING: """ ================================================ FILE: src/palimpzest/prompts/split_proposer_prompts.py ================================================ """This file contains prompts for SplitAndMerge operations.""" ### SYSTEM PROMPTS ### MAP_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a set of output fields to generate. Your task is to generate a detailed and succinct analysis describing what you believe is the correct value for each output field. Be sure to cite information from the context as evidence of why your answers are correct. Do not hallucinate evidence. You will be provided with a description of each input field and each output field. An example is shown below: --- INPUT FIELDS: {example_input_fields} OUTPUT FIELDS: {example_output_fields} CONTEXT: {{{example_context}}}{image_disclaimer}{audio_disclaimer} Let's think step-by-step in order to answer the question. ANSWER: {example_answer} --- """ FILTER_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a filter condition. Your task is to generate a detailed and succinct analysis describing whether you believe the input satisfies the filter condition. Be sure to cite information from the context as evidence of why your determination is correct. Do not hallucinate evidence. You will be provided with a description of each input field. An example is shown below: --- INPUT FIELDS: {example_input_fields} FILTER CONDITION: {example_filter_condition} CONTEXT: {{{example_context}}}{image_disclaimer}{audio_disclaimer} Let's think step-by-step in order to answer the question. ANSWER: {example_answer} --- """ ### USER / INSTANCE-SPECIFIC PROMPTS ### MAP_SPLIT_PROPOSER_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a set of output fields to generate. Your task is to generate a paragraph or two which describes what you believe is the correct value for each output field. Be sure to cite information from the context as evidence of why your answers are correct. Do not hallucinate evidence. {desc_section} You will be provided with a description of each input field and each output field. --- INPUT FIELDS: {input_fields_desc} OUTPUT FIELDS: {output_fields_desc} <>CONTEXT: {context}<> Let's think step-by-step in order to answer the question. ANSWER: """ FILTER_SPLIT_PROPOSER_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}. You will be presented with a context and a filter condition. Your task is to generate a detailed and succinct analysis describing whether you believe the input satisfies the filter condition. Be sure to cite information from the context as evidence of why your determination is correct. Do not hallucinate evidence. {desc_section} You will be provided with a description of each input field. An example is shown below: --- INPUT FIELDS: {input_fields_desc} FILTER CONDITION: {filter_condition} <>CONTEXT: {context}<> Let's think step-by-step in order to answer the question. ANSWER: """ ================================================ FILE: src/palimpzest/prompts/utils.py ================================================ """This file contains utility format strings which are templated into many of our prompts.""" ### FORMATTING INSTRUCTIONS ### ONE_TO_ONE_OUTPUT_FORMAT_INSTRUCTION = "Remember, your answer must be a valid JSON dictionary. The dictionary should only have the specified output fields." ONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION = "Remember, your answer must be a valid JSON list of dictionaries. The list may contain one or more dictionaries, and each dictionary should only have the specified output fields." ### USER-PROVIDED DESCRIPTION FOR MAPS / FILTERS / JOINS ### DESC_SECTION = """ The user has additionally provided you with this description of the task you need to perform: {desc} """ ### JOB INSTRUCTIONS ### AGG_JOB_INSTRUCTION = """analyze input {modalities} in order to perform an aggregation and generate a JSON object""" MAP_JOB_INSTRUCTION = """analyze input {modalities} in order to produce a JSON object""" FILTER_JOB_INSTRUCTION = """analyze input {modalities} in order to answer a TRUE / FALSE question""" JOIN_JOB_INSTRUCTION = """analyze input {modalities} in order to determine whether two data records satisfy a join condition""" PROPOSER_JOB_INSTRUCTION = """analyze input {modalities} in order to produce an answer to a question""" ### AGG / FILTER / JOIN CONDITIONS ### EXAMPLE_AGG_INSTRUCTION = "Count the distinct number of scientists in the input." EXAMPLE_FILTER_CONDITION = "The subject of the input is a foundational computer scientist." EXAMPLE_JOIN_CONDITION = "The two inputs are scientists in the same academic field." ### EXAMPLE INPUT FIELDS ### TEXT_EXAMPLE_INPUT_FIELDS = """ - text: a text passage describing a scientist - birthday: the scientist's birthday """ IMAGE_EXAMPLE_INPUT_FIELDS = """ - image: an image of the scientist - photographer: the photographer of the image """ AUDIO_EXAMPLE_INPUT_FIELDS = """ - recording: an audio recording of a newscast about the scientist's contributions to their field - speaker: the speaker in the recording """ RIGHT_TEXT_EXAMPLE_INPUT_FIELDS = """ - contents: the contents of a text file """ RIGHT_IMAGE_EXAMPLE_INPUT_FIELDS = """ - headshot: a headshot of a famous scientist """ RIGHT_AUDIO_EXAMPLE_INPUT_FIELDS = """ - podcast: an audio recording of a podcast about historic scientists """ ### EXAMPLE OUTPUT FIELDS ### TEXT_EXAMPLE_OUTPUT_FIELDS = """- name: the name of the scientist - birth_year: the year the scientist was born""" IMAGE_EXAMPLE_OUTPUT_FIELDS = """- is_bald: true if the scientist is bald and false otherwise""" AUDIO_EXAMPLE_OUTPUT_FIELDS = """- birthplace: the city where the scientist was born""" AGG_EXAMPLE_OUTPUT_FIELDS = """- num_distinct_scientists: the number of distinct scientists mentioned in the input""" ### EXAMPLE CONTEXTS ### TEXT_EXAMPLE_CONTEXT = """ "text": "Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace, was an English mathematician and writer chiefly known for her work on Charles Babbage's proposed mechanical general-purpose computer, the Analytical Engine. She was the first to recognise that the machine had applications beyond pure calculation.", "birthday": "December 10, 1815" """ IMAGE_EXAMPLE_CONTEXT = """ "image": , "photographer": "CameraEnthusiast1" """ AUDIO_EXAMPLE_CONTEXT = """ "recording": , "speaker": "Walter Cronkite" """ RIGHT_TEXT_EXAMPLE_CONTEXT = """ "content": "Alan Turing was a pioneering computer scientist and mathematician. He is widely considered to be the father of theoretical computer science and artificial intelligence." """ RIGHT_IMAGE_EXAMPLE_CONTEXT = """ "headshot": """ RIGHT_AUDIO_EXAMPLE_CONTEXT = """ "podcast": """ SECOND_TEXT_EXAMPLE_CONTEXT = """ "text": "Alan Turing was a pioneering computer scientist and mathematician. He is widely considered to be the father of theoretical computer science and artificial intelligence.", "birthday": "June 23, 1912" """ SECOND_IMAGE_EXAMPLE_CONTEXT = """ "image": , "photographer": "PhotoPro42" """ SECOND_AUDIO_EXAMPLE_CONTEXT = """ "recording": , "speaker": "Barbara Walters" """ THIRD_TEXT_EXAMPLE_CONTEXT = """ "text": "Ada Lovelace is a historically significant computer scientist.", "birthday": "December 10, 1815" """ THIRD_IMAGE_EXAMPLE_CONTEXT = """ "image": , "photographer": "PicturePerfect" """ THIRD_AUDIO_EXAMPLE_CONTEXT = """ "recording": , "speaker": "Anderson Cooper" """ ### DISCLAIMERS ### IMAGE_DISCLAIMER = """ \n """ AUDIO_DISCLAIMER = """ \n