[ { "path": ".github/workflows/ci.yaml", "content": "name: PZ Merge Checks\n\non:\n pull_request:\n branches:\n - main\n\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - name: Set up Python\n uses: actions/setup-python@v5\n with:\n python-version: '3.12'\n\n - name: Install dependencies\n run: |\n pip install --upgrade pip\n pip install .\n\n - name: Download and register testdata\n run: |\n pushd testdata\n wget -nc https://people.csail.mit.edu/gerarvit/PalimpzestData/enron-eval-tiny.tar.gz\n wget -nc https://people.csail.mit.edu/gerarvit/PalimpzestData/real-estate-eval-tiny.tar.gz\n tar -xzf enron-eval-tiny.tar.gz\n tar -xzf real-estate-eval-tiny.tar.gz\n rm *.tar.gz\n popd\n\n - name: Test with pytest\n env:\n OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}\n ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}\n run: |\n export CI=true\n export NO_GEMINI=true\n pip install pytest\n pytest -v tests/pytest\n\n lint-and-format:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - name: Set up Python\n uses: actions/setup-python@v5\n with:\n python-version: '3.x'\n - name: Install the code linting and formatting tool Ruff\n run: pip install \"ruff>=0.9.0\"\n - name: check version\n run: ruff --version\n - name: Lint code with Ruff\n run: ruff check --output-format=github --target-version=py38\n - name: Check code formatting with Ruff\n run: ruff check --no-fix . --target-version=py38\n continue-on-error: true\n\n check-version-bump:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - name: Check Version Increased\n run: |\n git fetch --prune --unshallow\n git checkout ${{ github.event.pull_request.base.sha }}\n VERSION=`cat pyproject.toml | grep '^version' | sed -E 's/version.*=.*\\\"(.*)\".*/\\1/'`\n echo \"Current version is $VERSION\"\n git checkout ${{ github.event.pull_request.head.sha }}\n VERSION_PR=`cat pyproject.toml | grep '^version' | sed -E 's/version.*=.*\\\"(.*)\".*/\\1/'`\n echo \"Version in PR is $VERSION_PR\"\n if [ \"$VERSION\" = \"$VERSION_PR\" ]; then\n echo \"Error: Version has not been bumped\"\n exit 1\n fi\n" }, { "path": ".github/workflows/docs.yaml", "content": "name: Deploy Docs to GitHub Pages\n\non:\n push:\n branches:\n - main\n\npermissions:\n contents: write\n\njobs:\n build:\n name: Build Docusaurus\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n - uses: actions/setup-node@v4\n with:\n node-version: 20\n cache: npm\n cache-dependency-path: website/package-lock.json\n\n - name: Install dependencies\n run: |\n cd website\n npm ci\n - name: Build website\n run: |\n cd website\n npm run build\n echo \"palimpzest.org\" > build/CNAME\n - name: Upload Build Artifact\n uses: actions/upload-pages-artifact@v3\n with:\n path: website/build\n\n deploy:\n name: Deploy to GitHub Pages\n needs: build\n\n # Grant GITHUB_TOKEN the permissions required to make a Pages deployment\n permissions:\n pages: write # to deploy to Pages\n id-token: write # to verify the deployment originates from an appropriate source\n\n # Deploy to the github-pages environment\n environment:\n name: github-pages\n url: ${{ steps.deployment.outputs.page_url }}\n\n runs-on: ubuntu-latest\n steps:\n - name: Deploy to GitHub Pages\n id: deployment\n uses: actions/deploy-pages@v4\n" }, { "path": ".github/workflows/package.yaml", "content": "name: package\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - name: Set up Python\n uses: actions/setup-python@v5\n with:\n python-version: '3.x'\n - name: Build Package\n run: |\n pip install --upgrade pip build\n python3 -m build\n - name: Store the distribution packages\n uses: actions/upload-artifact@v4\n with:\n name: python-package-distributions\n path: dist/\n\n publish:\n runs-on: ubuntu-latest\n name: Publish Package\n if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}\n needs:\n - build\n environment:\n name: pypi\n url: https://pypi.org/p/palimpzest\n permissions:\n id-token: write\n steps:\n - name: Download all the dists\n uses: actions/download-artifact@v4\n with:\n name: python-package-distributions\n path: dist/\n - name: Publish distribution to PyPI\n uses: pypa/gh-action-pypi-publish@release/v1\n\n github-release:\n name: >-\n Sign distribution w/Sigstore and upload to GitHub Release\n needs:\n - publish\n runs-on: ubuntu-latest\n permissions:\n contents: write\n id-token: write\n steps:\n - name: Download all the dists\n uses: actions/download-artifact@v4\n with:\n name: python-package-distributions\n path: dist/\n - name: Sign the dists with Sigstore\n uses: sigstore/gh-action-sigstore-python@v3.0.0\n with:\n inputs: >-\n ./dist/*.tar.gz\n ./dist/*.whl\n - name: Create GitHub Release\n env:\n GITHUB_TOKEN: ${{ github.token }}\n run: |\n PKG_VERSION=`ls dist/ | head -n 1 | sed -E 's/.*palimpzest-([0-9]+\\.[0-9]+\\.[0-9]+)-.*/\\1/'`\n gh release create \"$PKG_VERSION\" --repo \"$GITHUB_REPOSITORY\" --notes \"\"\n - name: Upload artifact signatures to GitHub Release\n env:\n GITHUB_TOKEN: ${{ github.token }}\n # Upload to GitHub Release using the `gh` CLI.\n # `dist/` contains the built packages, and the\n # sigstore-produced signatures and certificates.\n run: |\n PKG_VERSION=`ls dist/ | head -n 1 | sed -E 's/.*palimpzest-([0-9]+\\.[0-9]+\\.[0-9]+)-.*/\\1/'`\n gh release upload \"$PKG_VERSION\" dist/** --repo \"$GITHUB_REPOSITORY\"\n" }, { "path": ".github/workflows/test-docs.yaml", "content": "name: Test Building Docs\n\non:\n pull_request:\n branches:\n - main\n\njobs:\n test-deploy:\n name: Test deployment\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n - uses: actions/setup-node@v4\n with:\n node-version: 20\n cache: npm\n cache-dependency-path: website/package-lock.json\n\n - name: Install dependencies\n run: |\n cd website\n npm ci\n - name: Test build website\n run: |\n cd website\n npm run build\n" }, { "path": ".gitignore", "content": "docs/site/\n*.zip\n.cache/\n.env\nbuild/*\ndocs/build/*\ndocs/source/generated/*\ndist/*\n.vscode/*\n.idea/*\n.chroma\n.chroma-biodex\n.chroma-mmqa\n.ragatouille\nplots/\npaper-imgs/\n\n# testdata folders and archive files\ntestdata/enron-tiny.csv\ntestdata/*/\ntestdata/*.tar.gz\ntests/pytest/data/generator_messages/\nscripts/provider_stats/\nscripts/litellm_stats/\n\n# python artifacts\n*.egg-info\n**/__pycache__/\n\n# other\n.DS_Store\n\n# logs\n*.log\n\n# virtual environment(s)\nvenv/\nuv.lock\n\n# tmp\ntestdata/maildir/\ntestdata/real-estate-eval-100.tar\n\n# jupyter\n.ipynb_checkpoints/\n\n# evaluation\nold-eval-results/\neval-results/\n\ntestdata/enron-eval/*.txt\n\n# your zed using open source contributor who only installs in a virtual environment\n.venv\n.zed\npyrightconfig.json\n\nmyenv/\npz-env/\n\n# abacus-research data\nabacus-research/cuad-data/*\nabacus-research/opt-profiling-data/*\nabacus-research/parse-answer-errors/*\n\n# stats\nscripts/litellm_stats/\nscripts/provider_stats/\ntests/pytest/data/generator_messages/\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2024 MIT Data Systems Group\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "![pz-banner](https://palimpzest-workloads.s3.us-east-1.amazonaws.com/palimpzest-cropped.png)\n\n# Palimpzest (PZ)\n[![Discord](https://img.shields.io/discord/1245561987480420445?logo=discord)](https://discord.gg/dN85JJ6jaH)\n[![Docs](https://img.shields.io/badge/Read_the_Docs-purple?logo=readthedocs)](https://palimpzest.org/)\n[![Colab Demo](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1Fm8I4yL1az395MsFkQbEIZSmUZs0oGvZ?usp=sharing)\n[![PyPI](https://img.shields.io/pypi/v/palimpzest)](https://pypi.org/project/palimpzest/)\n[![PyPI - Monthly Downloads](https://img.shields.io/pypi/dm/palimpzest?color=teal)](https://pypi.org/project/palimpzest/)\n\n\n\n## 📚 Learn How to Use PZ\nOur [full documentation](https://palimpzest.org) is the definitive resource for learning how to use PZ. It contains all of the installation and quickstart materials on this page, as well as user guides, full API documentation (coming soon), and much more.\n\n## 🚀 Getting started\nYou can find a stable version of the PZ package on PyPI [here](https://pypi.org/project/palimpzest/). To install the package, run:\n```bash\n$ pip install palimpzest\n```\n\nYou can also install PZ with [uv](https://docs.astral.sh/uv/) for a faster installation:\n```bash\n$ uv pip install palimpzest\n```\n\nAlternatively, to install the latest version of the package from this repository, you can clone this repository and run the following commands:\n```bash\n$ git clone git@github.com:mitdbg/palimpzest.git\n$ cd palimpzest\n$ pip install .\n```\n\n## 🙋🏽 Join the PZ Community\nWe are actively hacking on PZ and would love to have you join our community [![Discord](https://img.shields.io/discord/1245561987480420445?logo=discord)](https://discord.gg/dN85JJ6jaH)\n\n[Our Discord server](https://discord.gg/dN85JJ6jaH) is the best place to:\n- Get help with your PZ program(s)\n- Give feedback to the maintainers\n- Discuss the future direction(s) of the project\n- Discuss anything related to data processing with LLMs!\n\nWe are eager to learn more about your workloads and use cases, and will take them into consideration in planning our future roadmap.\n\n### 📓 Citation\nIf you would like to cite our original paper on Palimpzest, please use the following citation:\n```\n@inproceedings{palimpzestCIDR,\n title={Palimpzest: Optimizing AI-Powered Analytics with Declarative Query Processing},\n author={Liu, Chunwei and Russo, Matthew and Cafarella, Michael and Cao, Lei and Chen, Peter Baile and Chen, Zui and Franklin, Michael and Kraska, Tim and Madden, Samuel and Shahout, Rana and Vitagliano, Gerardo},\n booktitle = {Proceedings of the {{Conference}} on {{Innovative Database Research}} ({{CIDR}})},\n date = 2025,\n}\n```\n\nIf you would like to cite our paper on Palimpzest's optimizer Abacus, please use the following citation:\n```\n@misc{russo2025abacuscostbasedoptimizersemantic,\n title={Abacus: A Cost-Based Optimizer for Semantic Operator Systems}, \n author={Matthew Russo and Sivaprasad Sudhir and Gerardo Vitagliano and Chunwei Liu and Tim Kraska and Samuel Madden and Michael Cafarella},\n year={2025},\n eprint={2505.14661},\n archivePrefix={arXiv},\n primaryClass={cs.DB},\n url={https://arxiv.org/abs/2505.14661}, \n}\n```\n" }, { "path": "abacus-research/README.md", "content": "## Chroma Embeddings and MMQA files\nYou can download the chroma embeddings we computed for MMQA and BioDEX by executing the following:\n```sh\n$ ./download_embeddings_and_mmqa.sh\n```\nThis folder also contains questions for the different splits of MMQA -- of which we only use `MMQA_dev.jsonl` for scoring PZ's output. If you need the full MMQA dataset for any reason (e.g. to visualize at which images are being retrieved by a pipeline), you can find it here: https://github.com/allenai/multimodalqa/tree/master.\n\n## Table 2\nThe following scripts create the data for Abacus in Table 2 in our Abacus paper.\n- `run_biodex.sh`\n- `run_cuad.sh`\n- `run_mmqa_complex.sh`\n\n## Table 3\nThe following scripts create the data for Abacus in Table 3 in our Abacus paper.\n- `run_biodex_min_cost_latency.sh`\n- `run_cuad_min_cost_latency.sh`\n- `run_mmqa_complex_min_cost_latency.sh`\n\n## Figure 6\nThe following scripts create the data for Figure 6 in our Abacus paper.\n- `run_biodex_priors.sh`\n- `run_biodex_priors_constrained.sh`\n- `run_cuad_priors.sh`\n- `run_cuad_priors_constrained.sh`\n\n## Figure 7\nThe `run_biodex_cost_threshold.sh` and `run_cuad_cost_threshold.sh` scripts create the data for Figure 6 in our Abacus paper.\n\n## Figure 8\nThe `run_ablation_study.sh` script creates the data for Figure 8 in our Abacus paper." }, { "path": "abacus-research/README_CUAD_LOCAL.md", "content": "# CUAD Local Data Setup and Usage\n\n## Setup\n\nSince HuggingFace datasets no longer supports loading scripts, we've created a local data loading solution.\n\n### 1. Download CUAD Data\n\nFirst, run the setup script to download CUAD data to a local directory:\n\n```bash\npython setup_cuad_data.py\n```\n\nThis will:\n- Create a `cuad-data/` directory\n- Download the CUAD dataset files (train and test JSON files)\n- Download the original dataset script from HuggingFace for reference\n\n### 2. Updated Scripts\n\nThe following scripts have been updated to use local data via `cuad_data_loader.py`:\n\n- **cuad-demo.py**\n- **cuad-max-quality-at-cost.py**\n\n### 3. Running the Scripts\n\n#### Basic CUAD Demo\n```bash\n# Make sure OPENAI_API_KEY is set in .env or environment\nsource ../.env && export OPENAI_API_KEY\n\n# Run from abacus-research directory\nseed=0\nexp_name=\"cuad-final-mab-k6-j4-budget50-seed${seed}\"\npython cuad-demo.py --k 6 --j 4 --sample-budget 50 --seed $seed --exp-name $exp_name --gpt4-mini-only\n```\n\n#### Max Quality at Cost\n```bash\npython cuad-max-quality-at-cost.py --constrained --gpt4-mini-only\n```" }, { "path": "abacus-research/biodex-ablation.py", "content": "import argparse\nimport json\nimport os\nimport time\n\nimport chromadb\nimport datasets\nfrom chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction\n\nimport palimpzest as pz\nfrom palimpzest.constants import Model\n\nbiodex_entry_cols = [\n {\"name\": \"pmid\", \"type\": str, \"desc\": \"The PubMed ID of the medical paper\"},\n {\"name\": \"title\", \"type\": str, \"desc\": \"The title of the medical paper\"},\n {\"name\": \"abstract\", \"type\": str, \"desc\": \"The abstract of the medical paper\"},\n {\"name\": \"fulltext\", \"type\": str, \"desc\": \"The full text of the medical paper, which contains information relevant for creating a drug safety report.\"},\n]\n\nbiodex_reactions_cols = [\n {\"name\": \"reactions\", \"type\": list[str], \"desc\": \"The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible.\"},\n]\n\nbiodex_reaction_labels_cols = [\n {\"name\": \"reaction_labels\", \"type\": list[str], \"desc\": \"Official terms for medical conditions listed in `reactions`\"},\n]\n\nbiodex_ranked_reactions_labels_cols = [\n {\"name\": \"ranked_reaction_labels\", \"type\": list[str], \"desc\": \"The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs.\"},\n]\n\nclass BiodexValidator(pz.Validator):\n def __init__(\n self,\n rp_at_k: int = 5,\n num_samples: int = 5,\n shuffle: bool = False,\n seed: int = 42,\n ):\n super().__init__()\n\n # read dataset and prepare entries\n dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=\"train\").to_pandas()\n if shuffle:\n dataset = dataset.sample(n=num_samples, random_state=seed).to_dict(orient=\"records\")\n else:\n dataset = dataset.to_dict(orient=\"records\")[:num_samples]\n\n # compute mapping from pmid --> label (i.e. reactions list)\n self.pmid_to_label = self._compute_pmid_to_label(dataset)\n\n # store rp_at_k for computing rank-precision at k metric\n self.k = rp_at_k\n\n def _compute_pmid_to_label(self, dataset: list[dict]) -> dict:\n \"\"\"Compute the label for a BioDEX report given its entry in the dataset.\"\"\"\n pmid_to_label = {}\n for entry in dataset:\n pmid = str(entry[\"pmid\"])\n reactions_lst = [\n reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n for reaction in entry[\"reactions\"].split(\",\")\n ]\n pmid_to_label[pmid] = reactions_lst\n\n return pmid_to_label\n\n def rank_precision_at_k(self, preds: list | None, targets: list):\n if preds is None:\n return 0.0\n\n try:\n # lower-case each list\n preds = [pred.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for pred in preds]\n targets = set([target.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for target in targets])\n\n # compute rank-precision at k\n rn = len(targets)\n denom = min(self.k, rn)\n total = 0.0\n for i in range(self.k):\n total += preds[i] in targets if i < len(preds) else 0.0\n\n return total / denom\n\n except Exception:\n os.makedirs(\"rp@k-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"rp@k-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def term_recall(self, preds: list | None, targets: list):\n if preds is None:\n return 0.0\n\n try:\n # normalize terms in each list\n pred_terms = set([\n term.strip()\n for pred in preds\n for term in pred.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n target_terms = ([\n term.strip()\n for target in targets\n for term in target.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n\n # compute term recall and return\n intersect = pred_terms.intersection(target_terms)\n term_recall = len(intersect) / len(target_terms)\n\n return term_recall\n\n except Exception:\n os.makedirs(\"term-recall-eval-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"term-recall-eval-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n field_name = fields[0]\n if field_name == \"reactions\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[str(input_record[\"pmid\"])]\n return self.term_recall(preds, targets)\n elif field_name == \"ranked_reaction_labels\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[str(input_record[\"pmid\"])]\n return self.rank_precision_at_k(preds, targets)\n else:\n raise NotImplementedError(f\"Validator.map_score_fn not implemented for field {field_name}.\")\n\n def topk_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n field_name = fields[0]\n if field_name == \"reaction_labels\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[input_record[\"pmid\"]]\n return self.term_recall(preds, targets)\n else:\n raise NotImplementedError(f\"Validator.topk_score_fn not implemented for field {field_name}.\")\n\n\nclass BiodexDataset(pz.IterDataset):\n def __init__(\n self,\n rp_at_k: int = 5,\n num_samples: int = 5,\n split: str = \"test\",\n shuffle: bool = False,\n seed: int = 42,\n ):\n super().__init__(id=\"biodex\", schema=biodex_entry_cols)\n\n self.dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=split).to_pandas()\n if shuffle:\n self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient=\"records\")\n else:\n self.dataset = self.dataset.to_dict(orient=\"records\")[:num_samples]\n\n self.rp_at_k = rp_at_k\n self.num_samples = num_samples\n self.shuffle = shuffle\n self.seed = seed\n self.split = split\n\n def __len__(self):\n return len(self.dataset)\n\n def __getitem__(self, idx: int):\n # get entry\n entry = self.dataset[idx]\n\n # get input fields\n pmid = entry[\"pmid\"]\n title = entry[\"title\"]\n abstract = entry[\"abstract\"]\n fulltext = entry[\"fulltext\"]\n\n # create item with fields\n item = {\"pmid\": pmid, \"title\": title, \"abstract\": abstract, \"fulltext\": fulltext}\n\n return item\n\n\nif __name__ == \"__main__\":\n # parse arguments\n parser = argparse.ArgumentParser(description=\"Run a simple demo\")\n parser.add_argument(\n \"--optimizer-strategy\",\n default=\"pareto\",\n type=str,\n help=\"The optimizer strategy to use. One of pareto or greedy\",\n )\n parser.add_argument(\n \"--seed\",\n default=42,\n type=int,\n help=\"Seed used to initialize RNG for MAB sampling algorithm\",\n )\n parser.add_argument(\n \"--k\",\n default=10,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--j\",\n default=3,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--sample-budget\",\n default=100,\n type=int,\n help=\"Total sample budget in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--exp-name\",\n default=None,\n type=str,\n help=\"The experiment name.\",\n )\n parser.add_argument(\n \"--policy\",\n default=None,\n type=str,\n help=\"The policy (one of 'mincost' or 'maxquality').\",\n )\n parser.add_argument(\n \"--priors-file\",\n default=None,\n type=str,\n help=\"A file with a dictionary mapping physical operator ids to prior belief on their performance\",\n )\n args = parser.parse_args()\n\n # create directory for profiling data\n os.makedirs(\"ablation-data\", exist_ok=True)\n\n seed = args.seed\n k = args.k\n j = args.j\n sample_budget = args.sample_budget\n optimizer_strategy = args.optimizer_strategy\n exp_name = args.exp_name\n priors = None\n if args.priors_file is not None and os.path.exists(args.priors_file):\n with open(args.priors_file) as f:\n priors = json.load(f)\n\n # set the optimization policy; constraint set to 80% of mean quality from unconstrained plans (Table 2)\n policy = (\n pz.MinCostAtFixedQuality(min_quality=0.8 * 0.261)\n if args.policy == \"mincost\"\n else pz.MaxQualityAtFixedCost(max_cost=0.5 * 0.7)\n )\n print(f\"USING POLICY: {policy}\")\n\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n # create validator\n validator = BiodexValidator(\n rp_at_k=5,\n num_samples=20,\n shuffle=True,\n seed=seed,\n )\n\n # create train dataset for validator\n train_dataset = BiodexDataset(\n split=\"train\",\n num_samples=20,\n shuffle=True,\n seed=seed,\n )\n train_dataset = {train_dataset.id: train_dataset}\n\n # load index [text-embedding-3-small]\n chroma_client = chromadb.PersistentClient(\".chroma-biodex\")\n openai_ef = OpenAIEmbeddingFunction(\n api_key=os.environ[\"OPENAI_API_KEY\"],\n model_name=\"text-embedding-3-small\",\n )\n index = chroma_client.get_collection(\"biodex-reaction-terms\", embedding_function=openai_ef)\n\n def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]:\n # execute query with embeddings\n results = index.query(query, n_results=5)\n\n # get list of result terms with their cosine similarity scores\n final_results = []\n for query_docs, query_distances in zip(results[\"documents\"], results[\"distances\"]):\n for doc, dist in zip(query_docs, query_distances):\n cosine_similarity = 1 - dist\n final_results.append({\"content\": doc, \"similarity\": cosine_similarity})\n\n # sort the results by similarity score\n sorted_results = sorted(final_results, key=lambda result: result[\"similarity\"], reverse=True)\n\n # remove duplicates\n sorted_results_set = set()\n final_sorted_results = []\n for result in sorted_results:\n if result[\"content\"] not in sorted_results_set:\n sorted_results_set.add(result[\"content\"])\n final_sorted_results.append(result[\"content\"])\n\n # return the top-k similar results and generation stats\n return {\"reaction_labels\": final_sorted_results[:k]}\n\n # construct plan\n plan = BiodexDataset(split=\"test\", num_samples=250, shuffle=True, seed=seed)\n plan = plan.sem_map(biodex_reactions_cols)\n plan = plan.sem_topk(\n index=index,\n search_func=search_func,\n search_attr=\"reactions\",\n output_attrs=biodex_reaction_labels_cols,\n )\n plan = plan.sem_map(biodex_ranked_reactions_labels_cols, depends_on=[\"title\", \"abstract\", \"fulltext\", \"reaction_labels\"])\n\n # set models\n models = [\n Model.GPT_4o,\n Model.GPT_4o_MINI,\n Model.LLAMA3_1_8B,\n Model.LLAMA3_3_70B,\n # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments`\n # Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B,\n ]\n\n # execute pz plan\n config = pz.QueryProcessorConfig(\n policy=policy,\n optimizer_strategy=optimizer_strategy,\n execution_strategy=\"parallel\",\n use_final_op_quality=True,\n max_workers=64,\n available_models=models,\n allow_bonded_query=True,\n allow_critic=True,\n allow_mixtures=True,\n allow_rag_reduction=True,\n progress=True,\n k=k,\n j=j,\n sample_budget=sample_budget,\n # sample_cost_budget=0.10,\n seed=seed,\n exp_name=exp_name,\n priors=priors,\n dont_use_priors=(priors is None),\n )\n\n data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator)\n\n print(data_record_collection.to_df())\n data_record_collection.to_df().to_csv(f\"ablation-data/{exp_name}-output.csv\", index=False)\n\n # create filepaths for records and stats\n records_path = f\"ablation-data/{exp_name}-records.json\"\n stats_path = f\"ablation-data/{exp_name}-profiling.json\"\n\n # save record outputs\n record_jsons = []\n for record in data_record_collection:\n record_dict = record.to_dict()\n record_dict = {\n k: v\n for k, v in record_dict.items()\n if k in [\"pmid\", \"reactions\", \"reaction_labels\", \"ranked_reaction_labels\"]\n }\n record_jsons.append(record_dict)\n\n with open(records_path, \"w\") as f:\n json.dump(record_jsons, f)\n\n # save statistics\n execution_stats_dict = data_record_collection.execution_stats.to_json()\n with open(stats_path, \"w\") as f:\n json.dump(execution_stats_dict, f)\n\n # score output\n test_dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=\"test\").to_pandas()\n test_dataset = test_dataset.sample(n=250, random_state=seed).to_dict(orient=\"records\")\n\n # construct mapping from pmid --> label (field, value) pairs\n def compute_target_record(entry):\n reactions_lst = [\n reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n for reaction in entry[\"reactions\"].split(\",\")\n ]\n label_dict = {\"ranked_reaction_labels\": reactions_lst}\n return label_dict\n\n label_fields_to_values = {\n entry[\"pmid\"]: compute_target_record(entry) for entry in test_dataset\n }\n\n def rank_precision_at_k(preds: list, targets: list, k: int):\n if preds is None:\n return 0.0\n\n # lower-case each list\n preds = [pred.lower().replace(\"'\", \"\").replace(\"^\", \"\") for pred in preds]\n targets = set([target.lower().replace(\"'\", \"\").replace(\"^\", \"\") for target in targets])\n\n # compute rank-precision at k\n rn = len(targets)\n denom = min(k, rn)\n total = 0.0\n for i in range(k):\n total += preds[i] in targets if i < len(preds) else 0.0\n\n return total / denom\n\n def compute_avg_rp_at_k(records, k=5):\n total_rp_at_k = 0\n bad = 0\n for record in records:\n pmid = record['pmid']\n preds = record['ranked_reaction_labels']\n targets = label_fields_to_values[pmid]['ranked_reaction_labels']\n try:\n total_rp_at_k += rank_precision_at_k(preds, targets, k)\n except Exception:\n bad += 1\n\n return total_rp_at_k / len(records), bad\n\n rp_at_k, bad = compute_avg_rp_at_k(record_jsons, k=5)\n final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0]\n final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id]\n stats_dict = {\n \"rp@5\": rp_at_k,\n \"optimization_time\": data_record_collection.execution_stats.optimization_time,\n \"optimization_cost\": data_record_collection.execution_stats.optimization_cost,\n \"plan_execution_time\": data_record_collection.execution_stats.plan_execution_time,\n \"plan_execution_cost\": data_record_collection.execution_stats.plan_execution_cost,\n \"total_execution_time\": data_record_collection.execution_stats.total_execution_time,\n \"total_execution_cost\": data_record_collection.execution_stats.total_execution_cost,\n \"plan_str\": final_plan_str,\n }\n with open(f\"ablation-data/{exp_name}-metrics.json\", \"w\") as f:\n json.dump(stats_dict, f)\n\n print(f\"bad: {bad}\")\n print(\"-------\")\n print(f\"rp@k: {rp_at_k:.5f}\")\n print(f\"Optimization time: {data_record_collection.execution_stats.optimization_time}\")\n print(f\"Optimization cost: {data_record_collection.execution_stats.optimization_cost}\")\n print(f\"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}\")\n print(f\"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}\")\n print(f\"Total Execution time: {data_record_collection.execution_stats.total_execution_time}\")\n print(f\"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}\")\n" }, { "path": "abacus-research/biodex-demo.py", "content": "import argparse\nimport json\nimport os\nimport time\n\nimport chromadb\nimport datasets\nfrom chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction\n\nimport palimpzest as pz\nfrom palimpzest.constants import Model\n\nbiodex_entry_cols = [\n {\"name\": \"pmid\", \"type\": str, \"desc\": \"The PubMed ID of the medical paper\"},\n {\"name\": \"title\", \"type\": str, \"desc\": \"The title of the medical paper\"},\n {\"name\": \"abstract\", \"type\": str, \"desc\": \"The abstract of the medical paper\"},\n {\"name\": \"fulltext\", \"type\": str, \"desc\": \"The full text of the medical paper, which contains information relevant for creating a drug safety report.\"},\n]\n\nbiodex_reactions_cols = [\n {\"name\": \"reactions\", \"type\": list[str], \"desc\": \"The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible.\"},\n]\n\nbiodex_reaction_labels_cols = [\n {\"name\": \"reaction_labels\", \"type\": list[str], \"desc\": \"Official terms for medical conditions listed in `reactions`\"},\n]\n\nbiodex_ranked_reactions_labels_cols = [\n {\"name\": \"ranked_reaction_labels\", \"type\": list[str], \"desc\": \"The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs.\"},\n]\n\nclass BiodexValidator(pz.Validator):\n def __init__(\n self,\n rp_at_k: int = 5,\n num_samples: int = 5,\n shuffle: bool = False,\n seed: int = 42,\n ):\n super().__init__()\n\n # read dataset and prepare entries\n dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=\"train\").to_pandas()\n if shuffle:\n dataset = dataset.sample(n=num_samples, random_state=seed).to_dict(orient=\"records\")\n else:\n dataset = dataset.to_dict(orient=\"records\")[:num_samples]\n\n # compute mapping from pmid --> label (i.e. reactions list)\n self.pmid_to_label = self._compute_pmid_to_label(dataset)\n\n # store rp_at_k for computing rank-precision at k metric\n self.k = rp_at_k\n\n def _compute_pmid_to_label(self, dataset: list[dict]) -> dict:\n \"\"\"Compute the label for a BioDEX report given its entry in the dataset.\"\"\"\n pmid_to_label = {}\n for entry in dataset:\n pmid = str(entry[\"pmid\"])\n reactions_lst = [\n reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n for reaction in entry[\"reactions\"].split(\",\")\n ]\n pmid_to_label[pmid] = reactions_lst\n\n return pmid_to_label\n\n def rank_precision_at_k(self, preds: list | None, targets: list):\n if preds is None:\n return 0.0\n\n try:\n # lower-case each list\n preds = [pred.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for pred in preds]\n targets = set([target.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for target in targets])\n\n # compute rank-precision at k\n rn = len(targets)\n denom = min(self.k, rn)\n total = 0.0\n for i in range(self.k):\n total += preds[i] in targets if i < len(preds) else 0.0\n\n return total / denom\n\n except Exception:\n os.makedirs(\"rp@k-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"rp@k-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def term_recall(self, preds: list | None, targets: list):\n if preds is None:\n return 0.0\n\n try:\n # normalize terms in each list\n pred_terms = set([\n term.strip()\n for pred in preds\n for term in pred.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n target_terms = ([\n term.strip()\n for target in targets\n for term in target.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n\n # compute term recall and return\n intersect = pred_terms.intersection(target_terms)\n term_recall = len(intersect) / len(target_terms)\n\n return term_recall\n\n except Exception:\n os.makedirs(\"term-recall-eval-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"term-recall-eval-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n field_name = fields[0]\n if field_name == \"reactions\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[str(input_record[\"pmid\"])]\n return self.term_recall(preds, targets)\n elif field_name == \"ranked_reaction_labels\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[str(input_record[\"pmid\"])]\n return self.rank_precision_at_k(preds, targets)\n else:\n raise NotImplementedError(f\"Validator.map_score_fn not implemented for field {field_name}.\")\n\n def topk_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n field_name = fields[0]\n if field_name == \"reaction_labels\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[input_record[\"pmid\"]]\n return self.term_recall(preds, targets)\n else:\n raise NotImplementedError(f\"Validator.topk_score_fn not implemented for field {field_name}.\")\n\n\nclass BiodexDataset(pz.IterDataset):\n def __init__(\n self,\n rp_at_k: int = 5,\n num_samples: int = 5,\n split: str = \"test\",\n shuffle: bool = False,\n seed: int = 42,\n ):\n super().__init__(id=\"biodex\", schema=biodex_entry_cols)\n\n self.dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=split).to_pandas()\n if shuffle:\n self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient=\"records\")\n else:\n self.dataset = self.dataset.to_dict(orient=\"records\")[:num_samples]\n\n self.rp_at_k = rp_at_k\n self.num_samples = num_samples\n self.shuffle = shuffle\n self.seed = seed\n self.split = split\n\n def __len__(self):\n return len(self.dataset)\n\n def __getitem__(self, idx: int):\n # get entry\n entry = self.dataset[idx]\n\n # get input fields\n pmid = entry[\"pmid\"]\n title = entry[\"title\"]\n abstract = entry[\"abstract\"]\n fulltext = entry[\"fulltext\"]\n\n # create item with fields\n item = {\"pmid\": pmid, \"title\": title, \"abstract\": abstract, \"fulltext\": fulltext}\n\n return item\n\n\nif __name__ == \"__main__\":\n # parse arguments\n parser = argparse.ArgumentParser(description=\"Run a simple demo\")\n parser.add_argument(\"--verbose\", default=False, action=\"store_true\", help=\"Print verbose output\")\n parser.add_argument(\"--progress\", default=False, action=\"store_true\", help=\"Print progress output\")\n parser.add_argument(\"--constrained\", default=False, action=\"store_true\", help=\"Use constrained objective\")\n parser.add_argument(\"--gpt4-mini-only\", default=False, action=\"store_true\", help=\"Use only GPT-4o-mini\")\n parser.add_argument(\n \"--execution-strategy\",\n default=\"parallel\",\n type=str,\n help=\"The plan executor to use. One of sequential, pipelined, parallel\",\n )\n parser.add_argument(\n \"--sentinel-execution-strategy\",\n default=\"mab\",\n type=str,\n help=\"The sentinel execution strategy to use. One of mab or random\",\n )\n parser.add_argument(\n \"--policy\",\n default=\"maxquality\",\n type=str,\n help=\"One of 'mincost', 'mintime', 'maxquality'\",\n )\n parser.add_argument(\n \"--val-examples\",\n default=25,\n type=int,\n help=\"Number of validation examples to sample from\",\n )\n parser.add_argument(\n \"--model\",\n default=\"gpt-4o\",\n type=str,\n help=\"One of 'gpt-4o', 'gpt-4o-mini', 'llama'\",\n )\n parser.add_argument(\n \"--seed\",\n default=42,\n type=int,\n help=\"Seed used to initialize RNG for MAB sampling algorithm\",\n )\n parser.add_argument(\n \"--k\",\n default=10,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--j\",\n default=3,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--sample-budget\",\n default=100,\n type=int,\n help=\"Total sample budget in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--exp-name\",\n default=None,\n type=str,\n help=\"The experiment name.\",\n )\n parser.add_argument(\n \"--priors-file\",\n default=None,\n type=str,\n help=\"A file with a dictionary mapping physical operator ids to prior belief on their performance\",\n )\n parser.add_argument(\n \"--quality\",\n default=None,\n type=float,\n help=\"Quality threshold\",\n )\n\n args = parser.parse_args()\n\n # create directory for profiling data\n os.makedirs(\"opt-profiling-data\", exist_ok=True)\n\n verbose = args.verbose\n progress = args.progress\n seed = args.seed\n val_examples = args.val_examples\n k = args.k\n j = args.j\n sample_budget = args.sample_budget\n execution_strategy = args.execution_strategy\n sentinel_execution_strategy = args.sentinel_execution_strategy\n exp_name = (\n f\"biodex-final-{sentinel_execution_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}\"\n if args.exp_name is None\n else args.exp_name\n )\n priors = None\n if args.priors_file is not None:\n with open(args.priors_file) as f:\n priors = json.load(f)\n\n # set the optimization policy; constraint set to 25% percentile from unconstrained plans\n policy = pz.MaxQuality() if not args.constrained else pz.MaxQualityAtFixedCost(max_cost=2.250)\n if args.policy == \"mincost\":\n policy = pz.MinCost()\n elif args.policy == \"minlatency\":\n policy = pz.MinTime()\n elif args.quality is not None and args.policy == \"mincostatfixedquality\":\n policy = pz.MinCostAtFixedQuality(min_quality=args.quality)\n elif args.quality is not None and args.policy == \"minlatencyatfixedquality\":\n policy = pz.MinTimeAtFixedQuality(min_quality=args.quality)\n print(f\"USING POLICY: {policy}\")\n\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n # create validator\n validator = BiodexValidator(\n rp_at_k=5,\n num_samples=val_examples,\n shuffle=True,\n seed=seed,\n )\n\n # create train dataset for validator\n train_dataset = BiodexDataset(\n split=\"train\",\n num_samples=val_examples,\n shuffle=True,\n seed=seed,\n )\n train_dataset = {train_dataset.id: train_dataset}\n\n # load index [text-embedding-3-small]\n chroma_client = chromadb.PersistentClient(\".chroma-biodex\")\n openai_ef = OpenAIEmbeddingFunction(\n api_key=os.environ[\"OPENAI_API_KEY\"],\n model_name=\"text-embedding-3-small\",\n )\n index = chroma_client.get_collection(\"biodex-reaction-terms\", embedding_function=openai_ef)\n\n def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]:\n # execute query with embeddings\n results = index.query(query, n_results=5)\n\n # get list of result terms with their cosine similarity scores\n final_results = []\n for query_docs, query_distances in zip(results[\"documents\"], results[\"distances\"]):\n for doc, dist in zip(query_docs, query_distances):\n cosine_similarity = 1 - dist\n final_results.append({\"content\": doc, \"similarity\": cosine_similarity})\n\n # sort the results by similarity score\n sorted_results = sorted(final_results, key=lambda result: result[\"similarity\"], reverse=True)\n\n # remove duplicates\n sorted_results_set = set()\n final_sorted_results = []\n for result in sorted_results:\n if result[\"content\"] not in sorted_results_set:\n sorted_results_set.add(result[\"content\"])\n final_sorted_results.append(result[\"content\"])\n\n # return the top-k similar results and generation stats\n return {\"reaction_labels\": final_sorted_results[:k]}\n\n # construct plan\n plan = BiodexDataset(split=\"test\", num_samples=250, shuffle=True, seed=seed)\n plan = plan.sem_map(biodex_reactions_cols)\n plan = plan.sem_topk(\n index=index,\n search_func=search_func,\n search_attr=\"reactions\",\n output_attrs=biodex_reaction_labels_cols,\n )\n plan = plan.sem_map(biodex_ranked_reactions_labels_cols, depends_on=[\"title\", \"abstract\", \"fulltext\", \"reaction_labels\"])\n\n # set models\n models = [Model.GPT_4o_MINI] if args.gpt4_mini_only else [\n Model.GPT_4o,\n Model.GPT_4o_MINI,\n Model.LLAMA3_1_8B,\n Model.LLAMA3_3_70B,\n # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments`\n Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B,\n ]\n\n # execute pz plan\n config = pz.QueryProcessorConfig(\n policy=policy,\n optimizer_strategy=\"pareto\",\n sentinel_execution_strategy=sentinel_execution_strategy,\n execution_strategy=execution_strategy,\n use_final_op_quality=True,\n max_workers=64,\n verbose=verbose,\n available_models=models,\n allow_bonded_query=True,\n allow_critic=True,\n allow_mixtures=True,\n allow_rag_reduction=True,\n progress=progress,\n k=k,\n j=j,\n sample_budget=sample_budget,\n # sample_cost_budget=0.10,\n seed=seed,\n exp_name=exp_name,\n priors=priors,\n )\n\n data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator)\n\n print(data_record_collection.to_df())\n data_record_collection.to_df().to_csv(f\"opt-profiling-data/{exp_name}-output.csv\", index=False)\n\n # create filepaths for records and stats\n records_path = f\"opt-profiling-data/{exp_name}-records.json\"\n stats_path = f\"opt-profiling-data/{exp_name}-profiling.json\"\n\n # save record outputs\n record_jsons = []\n for record in data_record_collection:\n record_dict = record.to_dict()\n record_dict = {\n k: v\n for k, v in record_dict.items()\n if k in [\"pmid\", \"reactions\", \"reaction_labels\", \"ranked_reaction_labels\"]\n }\n record_jsons.append(record_dict)\n\n with open(records_path, \"w\") as f:\n json.dump(record_jsons, f)\n\n # save statistics\n execution_stats_dict = data_record_collection.execution_stats.to_json()\n with open(stats_path, \"w\") as f:\n json.dump(execution_stats_dict, f)\n\n # score output\n test_dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=\"test\").to_pandas()\n test_dataset = test_dataset.sample(n=250, random_state=seed).to_dict(orient=\"records\")\n\n # construct mapping from pmid --> label (field, value) pairs\n def compute_target_record(entry):\n reactions_lst = [\n reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n for reaction in entry[\"reactions\"].split(\",\")\n ]\n label_dict = {\"ranked_reaction_labels\": reactions_lst}\n return label_dict\n\n label_fields_to_values = {\n entry[\"pmid\"]: compute_target_record(entry) for entry in test_dataset\n }\n\n def rank_precision_at_k(preds: list, targets: list, k: int):\n if preds is None:\n return 0.0\n\n # lower-case each list\n preds = [pred.lower().replace(\"'\", \"\").replace(\"^\", \"\") for pred in preds]\n targets = set([target.lower().replace(\"'\", \"\").replace(\"^\", \"\") for target in targets])\n\n # compute rank-precision at k\n rn = len(targets)\n denom = min(k, rn)\n total = 0.0\n for i in range(k):\n total += preds[i] in targets if i < len(preds) else 0.0\n\n return total / denom\n\n def compute_avg_rp_at_k(records, k=5):\n total_rp_at_k = 0\n bad = 0\n for record in records:\n pmid = record['pmid']\n preds = record['ranked_reaction_labels']\n targets = label_fields_to_values[pmid]['ranked_reaction_labels']\n try:\n total_rp_at_k += rank_precision_at_k(preds, targets, k)\n except Exception:\n bad += 1\n\n return total_rp_at_k / len(records), bad\n\n rp_at_k, bad = compute_avg_rp_at_k(record_jsons, k=5)\n final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0]\n final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id]\n stats_dict = {\n \"rp@5\": rp_at_k,\n \"optimization_time\": data_record_collection.execution_stats.optimization_time,\n \"optimization_cost\": data_record_collection.execution_stats.optimization_cost,\n \"plan_execution_time\": data_record_collection.execution_stats.plan_execution_time,\n \"plan_execution_cost\": data_record_collection.execution_stats.plan_execution_cost,\n \"total_execution_time\": data_record_collection.execution_stats.total_execution_time,\n \"total_execution_cost\": data_record_collection.execution_stats.total_execution_cost,\n \"plan_str\": final_plan_str,\n }\n with open(f\"opt-profiling-data/{exp_name}-metrics.json\", \"w\") as f:\n json.dump(stats_dict, f)\n\n print(f\"bad: {bad}\")\n print(\"-------\")\n print(f\"rp@k: {rp_at_k:.5f}\")\n print(f\"Optimization time: {data_record_collection.execution_stats.optimization_time}\")\n print(f\"Optimization cost: {data_record_collection.execution_stats.optimization_cost}\")\n print(f\"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}\")\n print(f\"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}\")\n print(f\"Total Execution time: {data_record_collection.execution_stats.total_execution_time}\")\n print(f\"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}\")\n" }, { "path": "abacus-research/biodex-max-quality-at-cost.py", "content": "import argparse\nimport json\nimport os\nimport time\n\nimport chromadb\nimport datasets\nfrom chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction\n\n# from ragatouille import RAGPretrainedModel\nimport palimpzest as pz\nfrom palimpzest.constants import Model\nfrom palimpzest.policy import MaxQuality, MaxQualityAtFixedCost\n\nbiodex_entry_cols = [\n {\"name\": \"pmid\", \"type\": str, \"desc\": \"The PubMed ID of the medical paper\"},\n {\"name\": \"title\", \"type\": str, \"desc\": \"The title of the medical paper\"},\n {\"name\": \"abstract\", \"type\": str, \"desc\": \"The abstract of the medical paper\"},\n {\"name\": \"fulltext\", \"type\": str, \"desc\": \"The full text of the medical paper, which contains information relevant for creating a drug safety report.\"},\n]\n\nbiodex_reactions_cols = [\n {\"name\": \"reactions\", \"type\": list[str], \"desc\": \"The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible.\"},\n]\n\nbiodex_reaction_labels_cols = [\n {\"name\": \"reaction_labels\", \"type\": list[str], \"desc\": \"Official terms for medical conditions listed in `reactions`\"},\n]\n\nbiodex_ranked_reactions_labels_cols = [\n {\"name\": \"ranked_reaction_labels\", \"type\": list[str], \"desc\": \"The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs.\"},\n]\n\nclass BiodexValidator(pz.Validator):\n def __init__(\n self,\n rp_at_k: int = 5,\n num_samples: int = 5,\n shuffle: bool = False,\n seed: int = 42,\n ):\n super().__init__()\n\n # read dataset and prepare entries\n dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=\"train\").to_pandas()\n if shuffle:\n dataset = dataset.sample(n=num_samples, random_state=seed).to_dict(orient=\"records\")\n else:\n dataset = dataset.to_dict(orient=\"records\")[:num_samples]\n\n # compute mapping from pmid --> label (i.e. reactions list)\n self.pmid_to_label = self._compute_pmid_to_label(dataset)\n\n # store rp_at_k for computing rank-precision at k metric\n self.k = rp_at_k\n\n def _compute_pmid_to_label(self, dataset: list[dict]) -> dict:\n \"\"\"Compute the label for a BioDEX report given its entry in the dataset.\"\"\"\n pmid_to_label = {}\n for entry in dataset:\n pmid = str(entry[\"pmid\"])\n reactions_lst = [\n reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n for reaction in entry[\"reactions\"].split(\",\")\n ]\n pmid_to_label[pmid] = reactions_lst\n\n return pmid_to_label\n\n def rank_precision_at_k(self, preds: list | None, targets: list):\n if preds is None:\n return 0.0\n\n try:\n # lower-case each list\n preds = [pred.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for pred in preds]\n targets = set([target.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for target in targets])\n\n # compute rank-precision at k\n rn = len(targets)\n denom = min(self.k, rn)\n total = 0.0\n for i in range(self.k):\n total += preds[i] in targets if i < len(preds) else 0.0\n\n return total / denom\n\n except Exception:\n os.makedirs(\"rp@k-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"rp@k-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def term_recall(self, preds: list | None, targets: list):\n if preds is None:\n return 0.0\n\n try:\n # normalize terms in each list\n pred_terms = set([\n term.strip()\n for pred in preds\n for term in pred.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n target_terms = ([\n term.strip()\n for target in targets\n for term in target.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n\n # compute term recall and return\n intersect = pred_terms.intersection(target_terms)\n term_recall = len(intersect) / len(target_terms)\n\n return term_recall\n\n except Exception:\n os.makedirs(\"term-recall-eval-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"term-recall-eval-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n field_name = fields[0]\n if field_name == \"reactions\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[str(input_record[\"pmid\"])]\n return self.term_recall(preds, targets)\n elif field_name == \"ranked_reaction_labels\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[str(input_record[\"pmid\"])]\n return self.rank_precision_at_k(preds, targets)\n else:\n raise NotImplementedError(f\"Validator.map_score_fn not implemented for field {field_name}.\")\n\n def topk_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n field_name = fields[0]\n if field_name == \"reaction_labels\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[input_record[\"pmid\"]]\n return self.term_recall(preds, targets)\n else:\n raise NotImplementedError(f\"Validator.topk_score_fn not implemented for field {field_name}.\")\n\n\nclass BiodexDataset(pz.IterDataset):\n def __init__(\n self,\n rp_at_k: int = 5,\n num_samples: int = 5,\n split: str = \"test\",\n shuffle: bool = False,\n seed: int = 42,\n ):\n super().__init__(id=\"biodex\", schema=biodex_entry_cols)\n\n self.dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=split).to_pandas()\n if shuffle:\n self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient=\"records\")\n else:\n self.dataset = self.dataset.to_dict(orient=\"records\")[:num_samples]\n\n self.rp_at_k = rp_at_k\n self.num_samples = num_samples\n self.shuffle = shuffle\n self.seed = seed\n self.split = split\n\n def __len__(self):\n return len(self.dataset)\n\n def __getitem__(self, idx: int):\n # get entry\n entry = self.dataset[idx]\n\n # get input fields\n pmid = entry[\"pmid\"]\n title = entry[\"title\"]\n abstract = entry[\"abstract\"]\n fulltext = entry[\"fulltext\"]\n\n # create item with fields\n item = {\"fields\": {}, \"labels\": {}, \"score_fn\": {}}\n item[\"fields\"][\"pmid\"] = pmid\n item[\"fields\"][\"title\"] = title\n item[\"fields\"][\"abstract\"] = abstract\n item[\"fields\"][\"fulltext\"] = fulltext\n\n return item\n\n\nif __name__ == \"__main__\":\n # parse arguments\n parser = argparse.ArgumentParser(description=\"Run a simple demo\")\n parser.add_argument(\"--verbose\", default=False, action=\"store_true\", help=\"Print verbose output\")\n parser.add_argument(\"--progress\", default=False, action=\"store_true\", help=\"Print progress output\")\n parser.add_argument(\n \"--execution-strategy\",\n default=\"parallel\",\n type=str,\n help=\"The plan executor to use. One of sequential, pipelined, parallel\",\n )\n parser.add_argument(\n \"--sentinel-execution-strategy\",\n default=\"mab\",\n type=str,\n help=\"The sentinel execution strategy to use. One of mab or random\",\n )\n parser.add_argument(\n \"--optimizer-strategy\",\n default=\"pareto\",\n type=str,\n help=\"The optimizer to use. One of pareto or greedy\",\n )\n parser.add_argument(\n \"--val-examples\",\n default=30,\n type=int,\n help=\"Number of validation examples to sample from\",\n )\n parser.add_argument(\n \"--seed\",\n default=42,\n type=int,\n help=\"Seed used to initialize RNG for MAB sampling algorithm\",\n )\n parser.add_argument(\n \"--k\",\n default=10,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--j\",\n default=3,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--sample-budget\",\n default=100,\n type=int,\n help=\"Total sample budget in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--cost\",\n default=1.0,\n type=float,\n help=\"The cost budget for the optimization\",\n )\n parser.add_argument(\n \"--exp-name\",\n default=None,\n type=str,\n help=\"The experiment name.\",\n )\n parser.add_argument(\n \"--priors-file\",\n default=None,\n type=str,\n help=\"A file with a dictionary mapping physical operator ids to prior belief on their performance\",\n )\n\n args = parser.parse_args()\n\n # create directory for profiling data\n os.makedirs(\"max-quality-at-cost-data\", exist_ok=True)\n\n verbose = args.verbose\n progress = args.progress\n seed = args.seed\n val_examples = args.val_examples\n k = args.k\n j = args.j\n sample_budget = args.sample_budget\n execution_strategy = args.execution_strategy\n sentinel_execution_strategy = args.sentinel_execution_strategy\n optimizer_strategy = args.optimizer_strategy\n cost = args.cost\n exp_name = (\n f\"biodex-strategy-{optimizer_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}\"\n if args.exp_name is None\n else args.exp_name\n )\n priors = None\n if args.priors_file is not None:\n with open(args.priors_file) as f:\n priors = json.load(f)\n print(f\"EXPERIMENT NAME: {exp_name}\")\n\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n # create validator\n validator = BiodexValidator(\n rp_at_k=5,\n num_samples=val_examples,\n shuffle=True,\n seed=seed,\n )\n\n # create validation data source\n train_dataset = BiodexDataset(\n split=\"train\",\n num_samples=val_examples,\n shuffle=True,\n seed=seed,\n )\n train_dataset = {train_dataset.id: train_dataset}\n\n # load index [text-embedding-3-small]\n chroma_client = chromadb.PersistentClient(\".chroma-biodex\")\n openai_ef = OpenAIEmbeddingFunction(\n api_key=os.environ[\"OPENAI_API_KEY\"],\n model_name=\"text-embedding-3-small\",\n )\n index = chroma_client.get_collection(\"biodex-reaction-terms\", embedding_function=openai_ef)\n\n def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]:\n # execute query with embeddings\n results = index.query(query, n_results=5)\n\n # get list of result terms with their cosine similarity scores\n final_results = []\n for query_docs, query_distances in zip(results[\"documents\"], results[\"distances\"]):\n for doc, dist in zip(query_docs, query_distances):\n cosine_similarity = 1 - dist\n final_results.append({\"content\": doc, \"similarity\": cosine_similarity})\n\n # sort the results by similarity score\n sorted_results = sorted(final_results, key=lambda result: result[\"similarity\"], reverse=True)\n\n # remove duplicates\n sorted_results_set = set()\n final_sorted_results = []\n for result in sorted_results:\n if result[\"content\"] not in sorted_results_set:\n sorted_results_set.add(result[\"content\"])\n final_sorted_results.append(result[\"content\"])\n\n # return the top-k similar results and generation stats\n return {\"reaction_labels\": final_sorted_results[:k]}\n\n # construct plan\n plan = BiodexDataset(split=\"test\", num_samples=250, shuffle=True, seed=seed)\n plan = plan.sem_map(biodex_reactions_cols)\n plan = plan.sem_topk(\n index=index,\n search_func=search_func,\n search_attr=\"reactions\",\n output_attrs=biodex_reaction_labels_cols,\n )\n plan = plan.sem_map(biodex_ranked_reactions_labels_cols, depends_on=[\"title\", \"abstract\", \"fulltext\", \"reaction_labels\"])\n\n # set policy\n policy = MaxQualityAtFixedCost(max_cost=cost) if cost < 999 else MaxQuality()\n\n # execute pz plan\n config = pz.QueryProcessorConfig(\n policy=policy,\n optimizer_strategy=optimizer_strategy,\n sentinel_execution_strategy=sentinel_execution_strategy,\n execution_strategy=execution_strategy,\n use_final_op_quality=True,\n max_workers=64,\n verbose=verbose,\n available_models=[\n Model.GPT_4o,\n Model.GPT_4o_MINI,\n Model.LLAMA3_1_8B,\n Model.LLAMA3_3_70B,\n # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments`\n Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B,\n ],\n allow_bonded_query=True,\n allow_critic=True,\n allow_mixtures=True,\n allow_rag_reduction=True,\n progress=progress,\n k=k,\n j=j,\n sample_budget=sample_budget,\n seed=seed,\n exp_name=exp_name,\n priors=priors,\n )\n\n data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator)\n\n print(data_record_collection.to_df())\n data_record_collection.to_df().to_csv(f\"max-quality-at-cost-data/{exp_name}-output.csv\", index=False)\n\n # create filepaths for records and stats\n records_path = f\"max-quality-at-cost-data/{exp_name}-records.json\"\n stats_path = f\"max-quality-at-cost-data/{exp_name}-profiling.json\"\n\n # save record outputs\n record_jsons = []\n for record in data_record_collection:\n record_dict = record.to_dict()\n record_dict = {\n k: v\n for k, v in record_dict.items()\n if k in [\"pmid\", \"reactions\", \"reaction_labels\", \"ranked_reaction_labels\"]\n }\n record_jsons.append(record_dict)\n\n with open(records_path, \"w\") as f:\n json.dump(record_jsons, f)\n\n # save statistics\n execution_stats_dict = data_record_collection.execution_stats.to_json()\n with open(stats_path, \"w\") as f:\n json.dump(execution_stats_dict, f)\n\n # score output\n test_dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=\"test\").to_pandas()\n test_dataset = test_dataset.sample(n=250, random_state=seed).to_dict(orient=\"records\")\n\n # construct mapping from pmid --> label (field, value) pairs\n def compute_target_record(entry):\n reactions_lst = [\n reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n for reaction in entry[\"reactions\"].split(\",\")\n ]\n label_dict = {\"ranked_reaction_labels\": reactions_lst}\n return label_dict\n\n label_fields_to_values = {\n entry[\"pmid\"]: compute_target_record(entry) for entry in test_dataset\n }\n\n def rank_precision_at_k(preds: list, targets: list, k: int):\n if preds is None:\n return 0.0\n\n # lower-case each list\n preds = [pred.lower().replace(\"'\", \"\").replace(\"^\", \"\") for pred in preds]\n targets = set([target.lower().replace(\"'\", \"\").replace(\"^\", \"\") for target in targets])\n\n # compute rank-precision at k\n rn = len(targets)\n denom = min(k, rn)\n total = 0.0\n for i in range(k):\n total += preds[i] in targets if i < len(preds) else 0.0\n\n return total / denom\n\n def compute_avg_rp_at_k(records, k=5):\n total_rp_at_k, bad = 0, 0\n for record in records:\n pmid = record['pmid']\n preds = record['ranked_reaction_labels']\n targets = label_fields_to_values[pmid]['ranked_reaction_labels']\n try:\n total_rp_at_k += rank_precision_at_k(preds, targets, k)\n except Exception:\n print(f\"Error computing rank precision at k for record with pmid {pmid}\")\n bad += 1\n\n return total_rp_at_k / len(records), bad\n\n rp_at_k, failed = compute_avg_rp_at_k(record_jsons, k=5)\n final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0]\n final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id]\n stats_dict = {\n \"rp@5\": rp_at_k,\n \"failed\": failed,\n \"optimization_time\": data_record_collection.execution_stats.optimization_time,\n \"optimization_cost\": data_record_collection.execution_stats.optimization_cost,\n \"plan_execution_time\": data_record_collection.execution_stats.plan_execution_time,\n \"plan_execution_cost\": data_record_collection.execution_stats.plan_execution_cost,\n \"total_execution_time\": data_record_collection.execution_stats.total_execution_time,\n \"total_execution_cost\": data_record_collection.execution_stats.total_execution_cost,\n \"plan_str\": final_plan_str,\n }\n with open(f\"max-quality-at-cost-data/{exp_name}-metrics.json\", \"w\") as f:\n json.dump(stats_dict, f)\n\n print(f\"rp@k: {rp_at_k:.5f}\")\n print(f\"failed: {failed}\")\n print(f\"Optimization time: {data_record_collection.execution_stats.optimization_time}\")\n print(f\"Optimization cost: {data_record_collection.execution_stats.optimization_cost}\")\n print(f\"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}\")\n print(f\"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}\")\n print(f\"Total Execution time: {data_record_collection.execution_stats.total_execution_time}\")\n print(f\"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}\")\n" }, { "path": "abacus-research/biodex-min-at-fixed-quality.py", "content": "import argparse\nimport json\nimport os\nimport time\n\nimport chromadb\nimport datasets\nfrom chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction\n\n# from ragatouille import RAGPretrainedModel\nimport palimpzest as pz\nfrom palimpzest.constants import Model\n\nbiodex_entry_cols = [\n {\"name\": \"pmid\", \"type\": str, \"desc\": \"The PubMed ID of the medical paper\"},\n {\"name\": \"title\", \"type\": str, \"desc\": \"The title of the medical paper\"},\n {\"name\": \"abstract\", \"type\": str, \"desc\": \"The abstract of the medical paper\"},\n {\"name\": \"fulltext\", \"type\": str, \"desc\": \"The full text of the medical paper, which contains information relevant for creating a drug safety report.\"},\n]\n\nbiodex_reactions_cols = [\n {\"name\": \"reactions\", \"type\": list[str], \"desc\": \"The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible.\"},\n]\n\nbiodex_reaction_labels_cols = [\n {\"name\": \"reaction_labels\", \"type\": list[str], \"desc\": \"Official terms for medical conditions listed in `reactions`\"},\n]\n\nbiodex_ranked_reactions_labels_cols = [\n {\"name\": \"ranked_reaction_labels\", \"type\": list[str], \"desc\": \"The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs.\"},\n]\n\nclass BiodexValidator(pz.Validator):\n def __init__(\n self,\n rp_at_k: int = 5,\n num_samples: int = 5,\n shuffle: bool = False,\n seed: int = 42,\n ):\n super().__init__()\n\n # read dataset and prepare entries\n dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=\"train\").to_pandas()\n if shuffle:\n dataset = dataset.sample(n=num_samples, random_state=seed).to_dict(orient=\"records\")\n else:\n dataset = dataset.to_dict(orient=\"records\")[:num_samples]\n\n # compute mapping from pmid --> label (i.e. reactions list)\n self.pmid_to_label = self._compute_pmid_to_label(dataset)\n\n # store rp_at_k for computing rank-precision at k metric\n self.k = rp_at_k\n\n def _compute_pmid_to_label(self, dataset: list[dict]) -> dict:\n \"\"\"Compute the label for a BioDEX report given its entry in the dataset.\"\"\"\n pmid_to_label = {}\n for entry in dataset:\n pmid = str(entry[\"pmid\"])\n reactions_lst = [\n reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n for reaction in entry[\"reactions\"].split(\",\")\n ]\n pmid_to_label[pmid] = reactions_lst\n\n return pmid_to_label\n\n def rank_precision_at_k(self, preds: list | None, targets: list):\n if preds is None:\n return 0.0\n\n try:\n # lower-case each list\n preds = [pred.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for pred in preds]\n targets = set([target.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for target in targets])\n\n # compute rank-precision at k\n rn = len(targets)\n denom = min(self.k, rn)\n total = 0.0\n for i in range(self.k):\n total += preds[i] in targets if i < len(preds) else 0.0\n\n return total / denom\n\n except Exception:\n os.makedirs(\"rp@k-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"rp@k-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def term_recall(self, preds: list | None, targets: list):\n if preds is None:\n return 0.0\n\n try:\n # normalize terms in each list\n pred_terms = set([\n term.strip()\n for pred in preds\n for term in pred.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n target_terms = ([\n term.strip()\n for target in targets\n for term in target.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n\n # compute term recall and return\n intersect = pred_terms.intersection(target_terms)\n term_recall = len(intersect) / len(target_terms)\n\n return term_recall\n\n except Exception:\n os.makedirs(\"term-recall-eval-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"term-recall-eval-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n field_name = fields[0]\n if field_name == \"reactions\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[str(input_record[\"pmid\"])]\n return self.term_recall(preds, targets)\n elif field_name == \"ranked_reaction_labels\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[str(input_record[\"pmid\"])]\n return self.rank_precision_at_k(preds, targets)\n else:\n raise NotImplementedError(f\"Validator.map_score_fn not implemented for field {field_name}.\")\n\n def topk_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n field_name = fields[0]\n if field_name == \"reaction_labels\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[input_record[\"pmid\"]]\n return self.term_recall(preds, targets)\n else:\n raise NotImplementedError(f\"Validator.topk_score_fn not implemented for field {field_name}.\")\n\n\nclass BiodexDataset(pz.IterDataset):\n def __init__(\n self,\n rp_at_k: int = 5,\n num_samples: int = 5,\n split: str = \"test\",\n shuffle: bool = False,\n seed: int = 42,\n ):\n super().__init__(id=\"biodex\", schema=biodex_entry_cols)\n\n self.dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=split).to_pandas()\n if shuffle:\n self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient=\"records\")\n else:\n self.dataset = self.dataset.to_dict(orient=\"records\")[:num_samples]\n\n self.rp_at_k = rp_at_k\n self.num_samples = num_samples\n self.shuffle = shuffle\n self.seed = seed\n self.split = split\n\n def __len__(self):\n return len(self.dataset)\n\n def __getitem__(self, idx: int):\n # get entry\n entry = self.dataset[idx]\n\n # get input fields\n pmid = entry[\"pmid\"]\n title = entry[\"title\"]\n abstract = entry[\"abstract\"]\n fulltext = entry[\"fulltext\"]\n\n # create item with fields\n item = {\"pmid\": pmid, \"title\": title, \"abstract\": abstract, \"fulltext\": fulltext}\n\n return item\n\n\nif __name__ == \"__main__\":\n # parse arguments\n parser = argparse.ArgumentParser(description=\"Run a simple demo\")\n parser.add_argument(\"--verbose\", default=False, action=\"store_true\", help=\"Print verbose output\")\n parser.add_argument(\"--progress\", default=False, action=\"store_true\", help=\"Print progress output\")\n parser.add_argument(\n \"--execution-strategy\",\n default=\"parallel\",\n type=str,\n help=\"The plan executor to use. One of sequential, pipelined, parallel\",\n )\n parser.add_argument(\n \"--sentinel-execution-strategy\",\n default=\"mab\",\n type=str,\n help=\"The sentinel execution strategy to use. One of mab or random\",\n )\n parser.add_argument(\n \"--optimizer-strategy\",\n default=\"pareto\",\n type=str,\n help=\"The optimizer to use. One of pareto or greedy\",\n )\n parser.add_argument(\n \"--val-examples\",\n default=30,\n type=int,\n help=\"Number of validation examples to sample from\",\n )\n parser.add_argument(\n \"--seed\",\n default=42,\n type=int,\n help=\"Seed used to initialize RNG for MAB sampling algorithm\",\n )\n parser.add_argument(\n \"--k\",\n default=10,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--j\",\n default=3,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--sample-budget\",\n default=100,\n type=int,\n help=\"Total sample budget in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--metric\",\n default=None,\n type=str,\n help=\"whether to minimize latency or cost\",\n )\n parser.add_argument(\n \"--exp-name\",\n default=None,\n type=str,\n help=\"The experiment name.\",\n )\n parser.add_argument(\n \"--priors-file\",\n default=None,\n type=str,\n help=\"A file with a dictionary mapping physical operator ids to prior belief on their performance\",\n )\n\n args = parser.parse_args()\n\n assert args.metric in [\"cost\", \"latency\"], \"metric must be one of cost or latency\"\n metric = args.metric\n\n # create directory for profiling data\n os.makedirs(f\"min-{metric}-at-quality-data\", exist_ok=True)\n\n verbose = args.verbose\n progress = args.progress\n seed = args.seed\n val_examples = args.val_examples\n k = args.k\n j = args.j\n sample_budget = args.sample_budget\n execution_strategy = args.execution_strategy\n sentinel_execution_strategy = args.sentinel_execution_strategy\n optimizer_strategy = args.optimizer_strategy\n exp_name = (\n f\"biodex-min-{metric}-strategy-{optimizer_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}\"\n if args.exp_name is None\n else args.exp_name\n )\n priors = None\n if args.priors_file is not None:\n with open(args.priors_file) as f:\n priors = json.load(f)\n print(f\"EXPERIMENT NAME: {exp_name}\")\n\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n # create validator\n validator = BiodexValidator(\n rp_at_k=5,\n num_samples=val_examples,\n shuffle=True,\n seed=seed,\n )\n\n # create validation data source\n train_dataset = BiodexDataset(\n split=\"train\",\n num_samples=val_examples,\n shuffle=True,\n seed=seed,\n )\n train_dataset = {train_dataset.id: train_dataset}\n\n # load index [text-embedding-3-small]\n chroma_client = chromadb.PersistentClient(\".chroma-biodex\")\n openai_ef = OpenAIEmbeddingFunction(\n api_key=os.environ[\"OPENAI_API_KEY\"],\n model_name=\"text-embedding-3-small\",\n )\n index = chroma_client.get_collection(\"biodex-reaction-terms\", embedding_function=openai_ef)\n\n def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]:\n # execute query with embeddings\n results = index.query(query, n_results=5)\n\n # get list of result terms with their cosine similarity scores\n final_results = []\n for query_docs, query_distances in zip(results[\"documents\"], results[\"distances\"]):\n for doc, dist in zip(query_docs, query_distances):\n cosine_similarity = 1 - dist\n final_results.append({\"content\": doc, \"similarity\": cosine_similarity})\n\n # sort the results by similarity score\n sorted_results = sorted(final_results, key=lambda result: result[\"similarity\"], reverse=True)\n\n # remove duplicates\n sorted_results_set = set()\n final_sorted_results = []\n for result in sorted_results:\n if result[\"content\"] not in sorted_results_set:\n sorted_results_set.add(result[\"content\"])\n final_sorted_results.append(result[\"content\"])\n\n # return the top-k similar results and generation stats\n return {\"reaction_labels\": final_sorted_results[:k]}\n\n # construct plan\n plan = BiodexDataset(split=\"test\", num_samples=250, shuffle=True, seed=seed)\n plan = plan.sem_map(biodex_reactions_cols)\n plan = plan.sem_topk(\n index=index,\n search_func=search_func,\n search_attr=\"reactions\",\n output_attrs=biodex_reaction_labels_cols,\n )\n plan = plan.sem_map(biodex_ranked_reactions_labels_cols, depends_on=[\"title\", \"abstract\", \"fulltext\", \"reaction_labels\"])\n\n # set policy\n policy = pz.MinCostAtFixedQuality(min_quality=0.216) if metric == \"cost\" else pz.MinTimeAtFixedQuality(min_quality=0.216)\n\n # execute pz plan\n config = pz.QueryProcessorConfig(\n policy=policy,\n optimizer_strategy=optimizer_strategy,\n sentinel_execution_strategy=sentinel_execution_strategy,\n execution_strategy=execution_strategy,\n use_final_op_quality=True,\n max_workers=64,\n verbose=verbose,\n available_models=[\n Model.GPT_4o_MINI,\n ],\n allow_bonded_query=True,\n allow_critic=True,\n allow_mixtures=True,\n allow_rag_reduction=True,\n progress=progress,\n k=k,\n j=j,\n sample_budget=sample_budget,\n seed=seed,\n exp_name=exp_name,\n priors=priors,\n )\n\n data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator)\n\n print(data_record_collection.to_df())\n data_record_collection.to_df().to_csv(f\"min-{metric}-at-quality-data/{exp_name}-output.csv\", index=False)\n\n # create filepaths for records and stats\n records_path = f\"min-{metric}-at-quality-data/{exp_name}-records.json\"\n stats_path = f\"min-{metric}-at-quality-data/{exp_name}-profiling.json\"\n\n # save record outputs\n record_jsons = []\n for record in data_record_collection:\n record_dict = record.to_dict()\n record_dict = {\n k: v\n for k, v in record_dict.items()\n if k in [\"pmid\", \"reactions\", \"reaction_labels\", \"ranked_reaction_labels\"]\n }\n record_jsons.append(record_dict)\n\n with open(records_path, \"w\") as f:\n json.dump(record_jsons, f)\n\n # save statistics\n execution_stats_dict = data_record_collection.execution_stats.to_json()\n with open(stats_path, \"w\") as f:\n json.dump(execution_stats_dict, f)\n\n # score output\n test_dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=\"test\").to_pandas()\n test_dataset = test_dataset.sample(n=250, random_state=seed).to_dict(orient=\"records\")\n\n # construct mapping from pmid --> label (field, value) pairs\n def compute_target_record(entry):\n reactions_lst = [\n reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n for reaction in entry[\"reactions\"].split(\",\")\n ]\n label_dict = {\"ranked_reaction_labels\": reactions_lst}\n return label_dict\n\n label_fields_to_values = {\n entry[\"pmid\"]: compute_target_record(entry) for entry in test_dataset\n }\n\n def rank_precision_at_k(preds: list, targets: list, k: int):\n if preds is None:\n return 0.0\n\n # lower-case each list\n preds = [pred.lower().replace(\"'\", \"\").replace(\"^\", \"\") for pred in preds]\n targets = set([target.lower().replace(\"'\", \"\").replace(\"^\", \"\") for target in targets])\n\n # compute rank-precision at k\n rn = len(targets)\n denom = min(k, rn)\n total = 0.0\n for i in range(k):\n total += preds[i] in targets if i < len(preds) else 0.0\n\n return total / denom\n\n def compute_avg_rp_at_k(records, k=5):\n total_rp_at_k, bad = 0, 0\n for record in records:\n pmid = record['pmid']\n preds = record['ranked_reaction_labels']\n targets = label_fields_to_values[pmid]['ranked_reaction_labels']\n try:\n total_rp_at_k += rank_precision_at_k(preds, targets, k)\n except Exception:\n print(f\"Error computing rank precision at k for record with pmid {pmid}\")\n bad += 1\n\n return total_rp_at_k / len(records), bad\n\n rp_at_k, failed = compute_avg_rp_at_k(record_jsons, k=5)\n final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0]\n final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id]\n stats_dict = {\n \"rp@5\": rp_at_k,\n \"failed\": failed,\n \"optimization_time\": data_record_collection.execution_stats.optimization_time,\n \"optimization_cost\": data_record_collection.execution_stats.optimization_cost,\n \"plan_execution_time\": data_record_collection.execution_stats.plan_execution_time,\n \"plan_execution_cost\": data_record_collection.execution_stats.plan_execution_cost,\n \"total_execution_time\": data_record_collection.execution_stats.total_execution_time,\n \"total_execution_cost\": data_record_collection.execution_stats.total_execution_cost,\n \"plan_str\": final_plan_str,\n }\n with open(f\"min-{metric}-at-quality-data/{exp_name}-metrics.json\", \"w\") as f:\n json.dump(stats_dict, f)\n\n print(f\"rp@k: {rp_at_k:.5f}\")\n print(f\"failed: {failed}\")\n print(f\"Optimization time: {data_record_collection.execution_stats.optimization_time}\")\n print(f\"Optimization cost: {data_record_collection.execution_stats.optimization_cost}\")\n print(f\"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}\")\n print(f\"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}\")\n print(f\"Total Execution time: {data_record_collection.execution_stats.total_execution_time}\")\n print(f\"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}\")\n" }, { "path": "abacus-research/biodex-pareto-cascades.py", "content": "import argparse\nimport json\nimport os\nimport time\n\nimport chromadb\nimport datasets\nfrom chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction\n\nimport palimpzest as pz\nfrom palimpzest.constants import Model\nfrom palimpzest.policy import MaxQualityAtFixedCost\n\nbiodex_entry_cols = [\n {\"name\": \"pmid\", \"type\": str, \"desc\": \"The PubMed ID of the medical paper\"},\n {\"name\": \"title\", \"type\": str, \"desc\": \"The title of the medical paper\"},\n {\"name\": \"abstract\", \"type\": str, \"desc\": \"The abstract of the medical paper\"},\n {\"name\": \"fulltext\", \"type\": str, \"desc\": \"The full text of the medical paper, which contains information relevant for creating a drug safety report.\"},\n]\n\nbiodex_reactions_cols = [\n {\"name\": \"reactions\", \"type\": list[str], \"desc\": \"The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible.\"},\n]\n\nbiodex_reaction_labels_cols = [\n {\"name\": \"reaction_labels\", \"type\": list[str], \"desc\": \"Official terms for medical conditions listed in `reactions`\"},\n]\n\nbiodex_ranked_reactions_labels_cols = [\n {\"name\": \"ranked_reaction_labels\", \"type\": list[str], \"desc\": \"The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs.\"},\n]\n\nclass BiodexValidator(pz.Validator):\n def __init__(\n self,\n rp_at_k: int = 5,\n num_samples: int = 5,\n shuffle: bool = False,\n seed: int = 42,\n ):\n super().__init__()\n\n # read dataset and prepare entries\n dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=\"train\").to_pandas()\n if shuffle:\n dataset = dataset.sample(n=num_samples, random_state=seed).to_dict(orient=\"records\")\n else:\n dataset = dataset.to_dict(orient=\"records\")[:num_samples]\n\n # compute mapping from pmid --> label (i.e. reactions list)\n self.pmid_to_label = self._compute_pmid_to_label(dataset)\n\n # store rp_at_k for computing rank-precision at k metric\n self.k = rp_at_k\n\n def _compute_pmid_to_label(self, dataset: list[dict]) -> dict:\n \"\"\"Compute the label for a BioDEX report given its entry in the dataset.\"\"\"\n pmid_to_label = {}\n for entry in dataset:\n pmid = str(entry[\"pmid\"])\n reactions_lst = [\n reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n for reaction in entry[\"reactions\"].split(\",\")\n ]\n pmid_to_label[pmid] = reactions_lst\n\n return pmid_to_label\n\n def rank_precision_at_k(self, preds: list | None, targets: list):\n if preds is None:\n return 0.0\n\n try:\n # lower-case each list\n preds = [pred.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for pred in preds]\n targets = set([target.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for target in targets])\n\n # compute rank-precision at k\n rn = len(targets)\n denom = min(self.k, rn)\n total = 0.0\n for i in range(self.k):\n total += preds[i] in targets if i < len(preds) else 0.0\n\n return total / denom\n\n except Exception:\n os.makedirs(\"rp@k-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"rp@k-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def term_recall(self, preds: list | None, targets: list):\n if preds is None:\n return 0.0\n\n try:\n # normalize terms in each list\n pred_terms = set([\n term.strip()\n for pred in preds\n for term in pred.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n target_terms = ([\n term.strip()\n for target in targets\n for term in target.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n\n # compute term recall and return\n intersect = pred_terms.intersection(target_terms)\n term_recall = len(intersect) / len(target_terms)\n\n return term_recall\n\n except Exception:\n os.makedirs(\"term-recall-eval-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"term-recall-eval-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n field_name = fields[0]\n if field_name == \"reactions\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[str(input_record[\"pmid\"])]\n return self.term_recall(preds, targets)\n elif field_name == \"ranked_reaction_labels\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[str(input_record[\"pmid\"])]\n return self.rank_precision_at_k(preds, targets)\n else:\n raise NotImplementedError(f\"Validator.map_score_fn not implemented for field {field_name}.\")\n\n def topk_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n field_name = fields[0]\n if field_name == \"reaction_labels\":\n preds = output.get(field_name)\n targets = self.pmid_to_label[input_record[\"pmid\"]]\n return self.term_recall(preds, targets)\n else:\n raise NotImplementedError(f\"Validator.topk_score_fn not implemented for field {field_name}.\")\n\n\nclass BiodexDataset(pz.IterDataset):\n def __init__(\n self,\n rp_at_k: int = 5,\n num_samples: int = 5,\n split: str = \"test\",\n shuffle: bool = False,\n seed: int = 42,\n ):\n super().__init__(id=\"biodex\", schema=biodex_entry_cols)\n\n self.dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=split).to_pandas()\n if shuffle:\n self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient=\"records\")\n else:\n self.dataset = self.dataset.to_dict(orient=\"records\")[:num_samples]\n\n self.rp_at_k = rp_at_k\n self.num_samples = num_samples\n self.shuffle = shuffle\n self.seed = seed\n self.split = split\n\n def __len__(self):\n return len(self.dataset)\n\n def __getitem__(self, idx: int):\n # get entry\n entry = self.dataset[idx]\n\n # get input fields\n pmid = entry[\"pmid\"]\n title = entry[\"title\"]\n abstract = entry[\"abstract\"]\n fulltext = entry[\"fulltext\"]\n\n # create item with fields\n item = {\"fields\": {}, \"labels\": {}, \"score_fn\": {}}\n item[\"fields\"][\"pmid\"] = pmid\n item[\"fields\"][\"title\"] = title\n item[\"fields\"][\"abstract\"] = abstract\n item[\"fields\"][\"fulltext\"] = fulltext\n\n return item\n\n\nif __name__ == \"__main__\":\n # parse arguments\n parser = argparse.ArgumentParser(description=\"Run a simple demo\")\n parser.add_argument(\"--verbose\", default=False, action=\"store_true\", help=\"Print verbose output\")\n parser.add_argument(\"--progress\", default=False, action=\"store_true\", help=\"Print progress output\")\n parser.add_argument(\"--constrained\", default=False, action=\"store_true\", help=\"Use constrained objective\")\n parser.add_argument(\n \"--execution-strategy\",\n default=\"parallel\",\n type=str,\n help=\"The plan executor to use. One of sequential, pipelined, parallel\",\n )\n parser.add_argument(\n \"--sentinel-execution-strategy\",\n default=\"mab\",\n type=str,\n help=\"The sentinel execution strategy to use. One of mab or random\",\n )\n parser.add_argument(\n \"--optimizer-strategy\",\n default=\"pareto\",\n type=str,\n help=\"The optimizer to use. One of pareto or greedy\",\n )\n parser.add_argument(\n \"--val-examples\",\n default=30,\n type=int,\n help=\"Number of validation examples to sample from\",\n )\n parser.add_argument(\n \"--seed\",\n default=42,\n type=int,\n help=\"Seed used to initialize RNG for MAB sampling algorithm\",\n )\n parser.add_argument(\n \"--k\",\n default=10,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--j\",\n default=3,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--sample-budget\",\n default=100,\n type=int,\n help=\"Total sample budget in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--cost\",\n default=1.0,\n type=float,\n help=\"The cost budget for the optimization\",\n )\n parser.add_argument(\n \"--exp-name\",\n default=None,\n type=str,\n help=\"The experiment name.\",\n )\n parser.add_argument(\n \"--priors-file\",\n default=None,\n type=str,\n help=\"A file with a dictionary mapping physical operator ids to prior belief on their performance\",\n )\n\n args = parser.parse_args()\n\n # create directory for profiling data\n os.makedirs(\"pareto-cascades-data\", exist_ok=True)\n\n verbose = args.verbose\n progress = args.progress\n seed = args.seed\n val_examples = args.val_examples\n k = args.k\n j = args.j\n sample_budget = args.sample_budget\n execution_strategy = args.execution_strategy\n sentinel_execution_strategy = args.sentinel_execution_strategy\n optimizer_strategy = args.optimizer_strategy\n cost = args.cost\n exp_name = (\n f\"biodex-strategy-{optimizer_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}\"\n if args.exp_name is None\n else args.exp_name\n )\n priors = None\n if args.priors_file is not None:\n with open(args.priors_file) as f:\n priors = json.load(f)\n print(f\"EXPERIMENT NAME: {exp_name}\")\n\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n # create validator\n validator = BiodexValidator(\n rp_at_k=5,\n num_samples=val_examples,\n shuffle=True,\n seed=seed,\n )\n\n # create validation data source\n train_dataset = BiodexDataset(\n split=\"train\",\n num_samples=val_examples,\n shuffle=True,\n seed=seed,\n )\n train_dataset = {train_dataset.id: train_dataset}\n\n # load index [text-embedding-3-small]\n chroma_client = chromadb.PersistentClient(\".chroma-biodex\")\n openai_ef = OpenAIEmbeddingFunction(\n api_key=os.environ[\"OPENAI_API_KEY\"],\n model_name=\"text-embedding-3-small\",\n )\n index = chroma_client.get_collection(\"biodex-reaction-terms\", embedding_function=openai_ef)\n\n def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]:\n # execute query with embeddings\n results = index.query(query, n_results=5)\n\n # get list of result terms with their cosine similarity scores\n final_results = []\n for query_docs, query_distances in zip(results[\"documents\"], results[\"distances\"]):\n for doc, dist in zip(query_docs, query_distances):\n cosine_similarity = 1 - dist\n final_results.append({\"content\": doc, \"similarity\": cosine_similarity})\n\n # sort the results by similarity score\n sorted_results = sorted(final_results, key=lambda result: result[\"similarity\"], reverse=True)\n\n # remove duplicates\n sorted_results_set = set()\n final_sorted_results = []\n for result in sorted_results:\n if result[\"content\"] not in sorted_results_set:\n sorted_results_set.add(result[\"content\"])\n final_sorted_results.append(result[\"content\"])\n\n # return the top-k similar results and generation stats\n return {\"reaction_labels\": final_sorted_results[:k]}\n\n # construct plan\n plan = BiodexDataset(split=\"test\", num_samples=250, shuffle=True, seed=seed)\n plan = plan.sem_map(biodex_reactions_cols)\n plan = plan.sem_topk(\n index=index,\n search_func=search_func,\n search_attr=\"reactions\",\n output_attrs=biodex_reaction_labels_cols,\n )\n plan = plan.sem_map(biodex_ranked_reactions_labels_cols, depends_on=[\"title\", \"abstract\", \"fulltext\", \"reaction_labels\"])\n\n # execute pz plan\n config = pz.QueryProcessorConfig(\n policy=MaxQualityAtFixedCost(max_cost=cost),\n optimizer_strategy=optimizer_strategy,\n sentinel_execution_strategy=sentinel_execution_strategy,\n execution_strategy=execution_strategy,\n use_final_op_quality=True,\n max_workers=64,\n verbose=verbose,\n available_models=[\n Model.GPT_4o_MINI,\n Model.LLAMA3_2_3B,\n Model.LLAMA3_1_8B,\n Model.LLAMA3_3_70B,\n # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments`\n Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B,\n ],\n allow_bonded_query=True,\n allow_critic=True,\n allow_mixtures=True,\n allow_rag_reduction=True,\n progress=progress,\n k=k,\n j=j,\n sample_budget=sample_budget,\n seed=seed,\n exp_name=exp_name,\n priors=priors,\n )\n\n data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator)\n\n print(data_record_collection.to_df())\n data_record_collection.to_df().to_csv(f\"pareto-cascades-data/{exp_name}-output.csv\", index=False)\n\n # create filepaths for records and stats\n records_path = f\"pareto-cascades-data/{exp_name}-records.json\"\n stats_path = f\"pareto-cascades-data/{exp_name}-profiling.json\"\n\n # save record outputs\n record_jsons = []\n for record in data_record_collection:\n record_dict = record.to_dict()\n record_dict = {\n k: v\n for k, v in record_dict.items()\n if k in [\"pmid\", \"reactions\", \"reaction_labels\", \"ranked_reaction_labels\"]\n }\n record_jsons.append(record_dict)\n\n with open(records_path, \"w\") as f:\n json.dump(record_jsons, f)\n\n # save statistics\n execution_stats_dict = data_record_collection.execution_stats.to_json()\n with open(stats_path, \"w\") as f:\n json.dump(execution_stats_dict, f)\n\n # score output\n test_dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=\"test\").to_pandas()\n test_dataset = test_dataset.sample(n=250, random_state=seed).to_dict(orient=\"records\")\n\n # construct mapping from pmid --> label (field, value) pairs\n def compute_target_record(entry):\n reactions_lst = [\n reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n for reaction in entry[\"reactions\"].split(\",\")\n ]\n label_dict = {\"ranked_reaction_labels\": reactions_lst}\n return label_dict\n\n label_fields_to_values = {\n entry[\"pmid\"]: compute_target_record(entry) for entry in test_dataset\n }\n\n def rank_precision_at_k(preds: list, targets: list, k: int):\n if preds is None:\n return 0.0\n\n # lower-case each list\n preds = [pred.lower().replace(\"'\", \"\").replace(\"^\", \"\") for pred in preds]\n targets = set([target.lower().replace(\"'\", \"\").replace(\"^\", \"\") for target in targets])\n\n # compute rank-precision at k\n rn = len(targets)\n denom = min(k, rn)\n total = 0.0\n for i in range(k):\n total += preds[i] in targets if i < len(preds) else 0.0\n\n return total / denom\n\n def compute_avg_rp_at_k(records, k=5):\n total_rp_at_k, bad = 0, 0\n for record in records:\n pmid = record['pmid']\n preds = record['ranked_reaction_labels']\n targets = label_fields_to_values[pmid]['ranked_reaction_labels']\n try:\n total_rp_at_k += rank_precision_at_k(preds, targets, k)\n except Exception:\n print(f\"Error computing rank precision at k for record with pmid {pmid}\")\n bad += 1\n\n return total_rp_at_k / len(records), bad\n\n rp_at_k, failed = compute_avg_rp_at_k(record_jsons, k=5)\n final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0]\n final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id]\n stats_dict = {\n \"rp@5\": rp_at_k,\n \"failed\": failed,\n \"optimization_time\": data_record_collection.execution_stats.optimization_time,\n \"optimization_cost\": data_record_collection.execution_stats.optimization_cost,\n \"plan_execution_time\": data_record_collection.execution_stats.plan_execution_time,\n \"plan_execution_cost\": data_record_collection.execution_stats.plan_execution_cost,\n \"total_execution_time\": data_record_collection.execution_stats.total_execution_time,\n \"total_execution_cost\": data_record_collection.execution_stats.total_execution_cost,\n \"plan_str\": final_plan_str,\n }\n with open(f\"pareto-cascades-data/{exp_name}-metrics.json\", \"w\") as f:\n json.dump(stats_dict, f)\n\n print(f\"rp@k: {rp_at_k:.5f}\")\n print(f\"failed: {failed}\")\n print(f\"Optimization time: {data_record_collection.execution_stats.optimization_time}\")\n print(f\"Optimization cost: {data_record_collection.execution_stats.optimization_cost}\")\n print(f\"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}\")\n print(f\"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}\")\n print(f\"Total Execution time: {data_record_collection.execution_stats.total_execution_time}\")\n print(f\"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}\")\n" }, { "path": "abacus-research/biodex-priors-cascades.json", "content": "{\"0005c18b69\": {\"quality\": 0.19444444444444442, \"cost\": 0.0038703809999999996, \"time\": 61.16110005378724}, \"009df798a3\": {\"quality\": 0.09252136752136753, \"cost\": 0.003526569, \"time\": 74.32940173149109}, \"00c93aec22\": {\"quality\": 0.10641025641025642, \"cost\": 0.010484405999999998, \"time\": 72.25670802593231}, \"00e1fecc4c\": {\"quality\": 0.21752136752136753, \"cost\": 0.006413474000000001, \"time\": 63.18833725452423}, \"00f4acd0d3\": {\"quality\": 0.0, \"cost\": 0.005918445, \"time\": 48.42981550693512}, \"01413aa72d\": {\"quality\": 0.16538461538461538, \"cost\": 0.008015024, \"time\": 69.29275906085968}, \"01c2f973ad\": {\"quality\": 0.1626068376068376, \"cost\": 0.002413815, \"time\": 42.94722969532013}, \"02078988c1\": {\"quality\": 0.0, \"cost\": 0.006904731999999999, \"time\": 27.019422554969786}, \"021604dec1\": {\"quality\": 0.3175213675213675, \"cost\": 0.008399656999999998, \"time\": 67.33289885520935}, \"02410c662e\": {\"quality\": 0.21752136752136753, \"cost\": 0.006360101999999999, \"time\": 77.91296794414521}, \"0262668df7\": {\"quality\": 0.21752136752136753, \"cost\": 0.0075904909999999996, \"time\": 59.82583842277527}, \"0267c97b70\": {\"quality\": 0.0, \"cost\": 0.003249014999999999, \"time\": 50.19215798377991}, \"02ae38e4aa\": {\"quality\": 0.1876068376068376, \"cost\": 0.004417125, \"time\": 69.9335319519043}, \"02f49fe0fd\": {\"quality\": 0.14807692307692308, \"cost\": 0.00044366999999999996, \"time\": 35.21025230884552}, \"030756558c\": {\"quality\": 0.04444444444444444, \"cost\": 0.002745915, \"time\": 53.435633063316345}, \"033ca325e6\": {\"quality\": 0.20277777777777778, \"cost\": 0.002408562, \"time\": 27.521768379211426}, \"038a5f0a62\": {\"quality\": 0.1876068376068376, \"cost\": 0.006526305, \"time\": 52.622441697120664}, \"041b5af43d\": {\"quality\": 0.21752136752136753, \"cost\": 0.010993874, \"time\": 64.76753988265992}, \"042d933706\": {\"quality\": 0.19444444444444442, \"cost\": 0.008023619, \"time\": 57.533840489387515}, \"04397effa0\": {\"quality\": 0.2064102564102564, \"cost\": 0.007009628999999999, \"time\": 76.10882368087769}, \"0539e0b42d\": {\"quality\": 0.25085470085470085, \"cost\": 0.007662647000000001, \"time\": 80.42491667270662}, \"0554568b86\": {\"quality\": 0.14038461538461539, \"cost\": 0.008002722, \"time\": 58.663866949081424}, \"06493715cc\": {\"quality\": 0.02222222222222222, \"cost\": 0.005358681, \"time\": 78.51178731918336}, \"067ee6e91b\": {\"quality\": 0.2098290598290598, \"cost\": 0.0072115949999999995, \"time\": 56.89639482498169}, \"068b66f00d\": {\"quality\": 0.1, \"cost\": 0.006528483999999999, \"time\": 68.53199887275696}, \"0695f9b5fc\": {\"quality\": 0.06944444444444445, \"cost\": 0.007583937, \"time\": 73.26675381660462}, \"073ed5b301\": {\"quality\": 0.07371794871794872, \"cost\": 0.005419403999999999, \"time\": 60.23991825580597}, \"079feb14a8\": {\"quality\": 0.2098290598290598, \"cost\": 0.008877133, \"time\": 62.487732887268066}, \"07a3a7daf7\": {\"quality\": 0.0, \"cost\": 0.006907794, \"time\": 43.434891891479495}, \"08127cd6dd\": {\"quality\": 0.1987179487179487, \"cost\": 0.012973673000000002, \"time\": 55.40713529586792}, \"0833133620\": {\"quality\": 0.1987179487179487, \"cost\": 0.005889005999999999, \"time\": 65.5436268568039}, \"089565077c\": {\"quality\": 0.025, \"cost\": 0.000339832, \"time\": 65.31151757240295}, \"08bf8cc191\": {\"quality\": 0.10555555555555557, \"cost\": 0.011634817000000002, \"time\": 95.24387204647064}, \"08e1802287\": {\"quality\": 0.23974358974358972, \"cost\": 0.0033798820000000003, \"time\": 25.597942876815797}, \"090cd3ef31\": {\"quality\": 0.23974358974358972, \"cost\": 0.0031741180000000005, \"time\": 25.463910150527955}, \"0944a921e8\": {\"quality\": 0.0702991452991453, \"cost\": 0.00792617, \"time\": 85.74197387695312}, \"0947216ece\": {\"quality\": 0.1987179487179487, \"cost\": 0.015770336, \"time\": 82.8155886888504}, \"096d51f670\": {\"quality\": 0.18482905982905984, \"cost\": 0.009268239, \"time\": 104.61172659397124}, \"09791c731b\": {\"quality\": 0.0438034188034188, \"cost\": 0.006413886, \"time\": 85.14133274555206}, \"0990c0d4f8\": {\"quality\": 0.2098290598290598, \"cost\": 0.007531591, \"time\": 77.474085521698}, \"0a128688c1\": {\"quality\": 0.18333333333333332, \"cost\": 0.014743780000000001, \"time\": 100.52884330749512}, \"0a4c1bbb4a\": {\"quality\": 0.16538461538461538, \"cost\": 0.005475987, \"time\": 77.0306186914444}, \"0ac969dde3\": {\"quality\": 0.19444444444444442, \"cost\": 0.009684573, \"time\": 102.05468263626099}, \"0af1efab0e\": {\"quality\": 0.0, \"cost\": 0.0035705999999999993, \"time\": 76.96206550598144}, \"0b1ed7ff58\": {\"quality\": 0.19252136752136753, \"cost\": 0.008049169, \"time\": 92.94120838642121}, \"0b3dc2e896\": {\"quality\": 0.0, \"cost\": 0.00892118, \"time\": 96.49321339130401}, \"0b43e94f3f\": {\"quality\": 0.1987179487179487, \"cost\": 0.001608864, \"time\": 40.973159313201904}, \"0b4ab72197\": {\"quality\": 0.061111111111111116, \"cost\": 0.010094982, \"time\": 72.49727232456206}, \"0be862a0dc\": {\"quality\": 0.04038461538461539, \"cost\": 0.0062549730000000005, \"time\": 56.527322101593015}, \"0bf3129ae8\": {\"quality\": 0.16538461538461538, \"cost\": 0.0028206660000000003, \"time\": 54.240378093719485}, \"0c020b86a3\": {\"quality\": 0.18482905982905984, \"cost\": 0.008657178000000001, \"time\": 49.74211900234222}, \"0c6c7fe96a\": {\"quality\": 0.12863247863247865, \"cost\": 0.012195626999999999, \"time\": 80.83456852436066}, \"0c81c8996a\": {\"quality\": 0.023076923076923078, \"cost\": 0.006098082000000001, \"time\": 71.85198886394501}, \"0cdc5954dd\": {\"quality\": 0.11666666666666667, \"cost\": 0.008310576, \"time\": 66.0079866170883}, \"0d25188bf7\": {\"quality\": 0.04807692307692308, \"cost\": 0.008176553999999999, \"time\": 84.21552875041962}, \"0d9d767ae5\": {\"quality\": 0.04444444444444444, \"cost\": 0.012340264, \"time\": 71.0254203081131}, \"0e36342fe7\": {\"quality\": 0.09252136752136753, \"cost\": 0.00467736, \"time\": 38.08519749641418}, \"0e7e862290\": {\"quality\": 0.0, \"cost\": 0.003672576, \"time\": 74.36907055377961}, \"0e91cd07f9\": {\"quality\": 0.18333333333333332, \"cost\": 0.006350159999999999, \"time\": 76.38928816318511}, \"0ec672e7c8\": {\"quality\": 0.2098290598290598, \"cost\": 0.013312296999999999, \"time\": 90.61400089263915}, \"0ed243f788\": {\"quality\": 0.23205128205128206, \"cost\": 0.008812347, \"time\": 80.54674921035766}, \"0eeb372802\": {\"quality\": 0.23974358974358972, \"cost\": 0.013314724, \"time\": 94.53299105167389}, \"0effe9b1dc\": {\"quality\": 0.15918803418803418, \"cost\": 0.003532296, \"time\": 72.70150463581085}, \"0f7faf684d\": {\"quality\": 0.10982905982905983, \"cost\": 0.011855149000000002, \"time\": 89.06640274524689}, \"0fcec544e3\": {\"quality\": 0.25982905982905985, \"cost\": 0.004687262999999999, \"time\": 83.44144034385681}, \"0ff126ebf8\": {\"quality\": 0.21752136752136753, \"cost\": 0.011823445, \"time\": 64.27427606582641}, \"112d9a3421\": {\"quality\": 0.2730769230769231, \"cost\": 0.010725950000000001, \"time\": 87.46276273727418}, \"114a097c53\": {\"quality\": 0.10982905982905983, \"cost\": 0.00542718, \"time\": 47.13481698036194}, \"116334cd72\": {\"quality\": 0.3175213675213675, \"cost\": 0.012438881999999998, \"time\": 93.07386040687561}, \"1175ee37e6\": {\"quality\": 0.19444444444444442, \"cost\": 0.001459101, \"time\": 30.085742378234862}, \"11a66478dc\": {\"quality\": 0.125, \"cost\": 0.00746376, \"time\": 63.90720744132996}, \"11bc996d48\": {\"quality\": 0.2098290598290598, \"cost\": 0.0065042699999999995, \"time\": 65.52873225212097}, \"11debf9fc0\": {\"quality\": 0.18482905982905984, \"cost\": 0.006505245, \"time\": 89.37637939453126}, \"123fb650fb\": {\"quality\": 0.21752136752136753, \"cost\": 0.008327412, \"time\": 88.56314113140107}, \"1274c21076\": {\"quality\": 0.125, \"cost\": 0.011184342, \"time\": 102.6374900341034}, \"127af50739\": {\"quality\": 0.16538461538461538, \"cost\": 0.008449134, \"time\": 104.71919329166413}, \"12addbf5e2\": {\"quality\": 0.04444444444444444, \"cost\": 0.003276834, \"time\": 88.02170798778533}, \"133ee5023f\": {\"quality\": 0.1987179487179487, \"cost\": 0.005588716, \"time\": 53.45017175674438}, \"1368e1c78e\": {\"quality\": 0.16752136752136754, \"cost\": 0.0034759379999999996, \"time\": 84.43210818767548}, \"13a009fe0c\": {\"quality\": 0.21752136752136753, \"cost\": 0.012945674, \"time\": 98.5892866373062}, \"13da306f84\": {\"quality\": 0.10363247863247865, \"cost\": 0.0087151, \"time\": 77.5059399843216}, \"13f2b9c25b\": {\"quality\": 0.25982905982905985, \"cost\": 0.003494103, \"time\": 98.82596600055695}, \"13f75f9bd0\": {\"quality\": 0.2064102564102564, \"cost\": 0.007151064, \"time\": 70.01372668743133}, \"1404e0aa35\": {\"quality\": 0.2098290598290598, \"cost\": 0.007724231, \"time\": 94.99440484046936}, \"140ededb41\": {\"quality\": 0.2098290598290598, \"cost\": 0.001514445, \"time\": 19.843628692626954}, \"142f3a7c70\": {\"quality\": 0.0, \"cost\": 0.004404129, \"time\": 77.21951529979705}, \"1468dddecc\": {\"quality\": 0.16944444444444445, \"cost\": 0.002386455, \"time\": 48.906515979766844}, \"14d19a01e2\": {\"quality\": 0.05641025641025641, \"cost\": 0.0063798959999999995, \"time\": 72.75729236602783}, \"1624bb5302\": {\"quality\": 0.18482905982905984, \"cost\": 0.001919664, \"time\": 66.44049925804138}, \"1636e0833b\": {\"quality\": 0.025, \"cost\": 0.002752764, \"time\": 74.27101521492004}, \"1658296f3a\": {\"quality\": 0.21752136752136753, \"cost\": 0.005375874, \"time\": 64.17133255004883}, \"16f351273f\": {\"quality\": 0.17307692307692307, \"cost\": 0.01028627, \"time\": 67.0687807559967}, \"171e6ae293\": {\"quality\": 0.21752136752136753, \"cost\": 0.007912404000000001, \"time\": 62.13353226184845}, \"176da24f53\": {\"quality\": 0.015384615384615385, \"cost\": 0.0038477320000000004, \"time\": 42.74407241344451}, \"179379555f\": {\"quality\": 0.1987179487179487, \"cost\": 0.006129615, \"time\": 88.43889818191528}, \"17c928174f\": {\"quality\": 0.1987179487179487, \"cost\": 0.0028421130000000003, \"time\": 75.59559867382049}, \"181c91d1be\": {\"quality\": 0.21752136752136753, \"cost\": 0.012095439999999999, \"time\": 67.77491972446441}, \"18368684cd\": {\"quality\": 0.05641025641025641, \"cost\": 0.0059048880000000005, \"time\": 90.52595224380494}, \"183743e76e\": {\"quality\": 0.0, \"cost\": 0.005899368, \"time\": 74.72685542106629}, \"18f55750b0\": {\"quality\": 0.0, \"cost\": 0.008405136, \"time\": 124.15524208545685}, \"19563b057d\": {\"quality\": 0.1987179487179487, \"cost\": 0.011058744, \"time\": 94.49535017013551}, \"1957127275\": {\"quality\": 0.29252136752136754, \"cost\": 0.008193474000000001, \"time\": 92.91692547798156}, \"197bb53f10\": {\"quality\": 0.2098290598290598, \"cost\": 0.006091916999999999, \"time\": 79.83643505573272}, \"199fd1fbf2\": {\"quality\": 0.058333333333333334, \"cost\": 0.00121965, \"time\": 48.699701046943666}, \"19b40e0271\": {\"quality\": 0.3175213675213675, \"cost\": 0.008674995000000001, \"time\": 116.94037146568297}, \"19e3db7fe7\": {\"quality\": 0.025, \"cost\": 0.01204651, \"time\": 117.68800120353698}, \"1ad856985f\": {\"quality\": 0.19444444444444442, \"cost\": 0.008808637000000001, \"time\": 115.93177556991577}, \"1adec2dca2\": {\"quality\": 0.0, \"cost\": 0.013760218000000001, \"time\": 125.3757052898407}, \"1b04a2b184\": {\"quality\": 0.09252136752136753, \"cost\": 0.013170014, \"time\": 111.55746636390685}, \"1b28439bd7\": {\"quality\": 0.21752136752136753, \"cost\": 0.010848586, \"time\": 84.68317291736602}, \"1beb2fac62\": {\"quality\": 0.3175213675213675, \"cost\": 0.013538090000000001, \"time\": 73.24731080532074}, \"1c347e4d91\": {\"quality\": 0.11752136752136753, \"cost\": 0.003958569, \"time\": 81.87744581699371}, \"1c3882926e\": {\"quality\": 0.06538461538461539, \"cost\": 0.006650534999999999, \"time\": 102.02758514881134}, \"1cc6d9efb6\": {\"quality\": 0.21752136752136753, \"cost\": 0.011642267999999999, \"time\": 73.37450892925263}, \"1ce3d77039\": {\"quality\": 0.1987179487179487, \"cost\": 0.000591721, \"time\": 33.82067358493805}, \"1ce99cf2c8\": {\"quality\": 0.19252136752136753, \"cost\": 0.007676729, \"time\": 109.94239032268524}, \"1d26090364\": {\"quality\": 0.08141025641025641, \"cost\": 0.004802969999999999, \"time\": 37.3655684709549}, \"1d87f97e62\": {\"quality\": 0.25085470085470085, \"cost\": 0.0074821539999999995, \"time\": 69.90373904705048}, \"1da2369719\": {\"quality\": 0.1737179487179487, \"cost\": 0.003804339, \"time\": 75.42430078983307}, \"1e18e60895\": {\"quality\": 0.1952991452991453, \"cost\": 0.000637105, \"time\": 36.58538358211517}, \"1e1bf7e88b\": {\"quality\": 0.2064102564102564, \"cost\": 0.012300497, \"time\": 91.10763094425201}, \"1e8b3521f8\": {\"quality\": 0.23205128205128206, \"cost\": 0.003055887, \"time\": 85.00319654941559}, \"1f5e8c9e9a\": {\"quality\": 0.06538461538461539, \"cost\": 0.01254384, \"time\": 110.50039112567902}, \"2018bef45f\": {\"quality\": 0.13974358974358975, \"cost\": 0.0073448359999999996, \"time\": 90.09606876373292}, \"2066966577\": {\"quality\": 0.19444444444444442, \"cost\": 0.011873376, \"time\": 119.55739791393279}, \"2075ff1d04\": {\"quality\": 0.08333333333333334, \"cost\": 0.0009685659999999999, \"time\": 70.29870116710663}, \"2080b60a57\": {\"quality\": 0.04444444444444444, \"cost\": 0.005072787, \"time\": 116.32027242183685}, \"208a98f514\": {\"quality\": 0.21752136752136753, \"cost\": 0.009998995, \"time\": 118.32991974353791}, \"20e10af7d4\": {\"quality\": 0.2098290598290598, \"cost\": 0.005652002999999999, \"time\": 116.42957108020784}, \"20e2c0b057\": {\"quality\": 0.10982905982905983, \"cost\": 0.007542789, \"time\": 113.53230805397033}, \"211b89b4cd\": {\"quality\": 0.1952991452991453, \"cost\": 0.005615415, \"time\": 63.006171917915346}, \"2153174e2d\": {\"quality\": 0.11474358974358972, \"cost\": 0.007281448, \"time\": 118.94957902431489}, \"21b2b8ebd1\": {\"quality\": 0.21752136752136753, \"cost\": 0.007063268000000001, \"time\": 76.91416993141175}, \"21b2df8512\": {\"quality\": 0.19444444444444442, \"cost\": 0.00326791, \"time\": 69.47645666599274}, \"21bed16a7d\": {\"quality\": 0.07307692307692308, \"cost\": 0.0016491099999999999, \"time\": 30.10742793083191}, \"227246dff8\": {\"quality\": 0.1876068376068376, \"cost\": 0.0006464099999999999, \"time\": 42.72886557579041}, \"227c30d349\": {\"quality\": 0.1, \"cost\": 0.005918874, \"time\": 103.58872454166413}, \"228687831a\": {\"quality\": 0.04722222222222222, \"cost\": 0.005076029999999999, \"time\": 67.8051172733307}, \"23075b2a6e\": {\"quality\": 0.05, \"cost\": 0.0024716549999999997, \"time\": 97.33722229003905}, \"23566f15ab\": {\"quality\": 0.1876068376068376, \"cost\": 0.00044987499999999997, \"time\": 40.82834756374359}, \"2370cebb10\": {\"quality\": 0.21752136752136753, \"cost\": 0.004052682, \"time\": 106.21984114646912}, \"2386b03c4c\": {\"quality\": 0.1987179487179487, \"cost\": 0.006683786999999999, \"time\": 110.57667918205262}, \"24957f3a43\": {\"quality\": 0.1987179487179487, \"cost\": 0.004216121999999999, \"time\": 70.75154151916504}, \"24c122de4e\": {\"quality\": 0.061111111111111116, \"cost\": 0.0059287260000000005, \"time\": 78.56412732601166}, \"24f76747b9\": {\"quality\": 0.14871794871794872, \"cost\": 0.003334974, \"time\": 96.20667114257813}, \"2609bfd616\": {\"quality\": 0.08482905982905983, \"cost\": 0.009335312, \"time\": 101.31035015583038}, \"260ab3e966\": {\"quality\": 0.18482905982905984, \"cost\": 0.006512652000000001, \"time\": 107.31265118122101}, \"2728c8eb6a\": {\"quality\": 0.05918803418803419, \"cost\": 0.009020646, \"time\": 82.33452692031861}, \"27971eaaf5\": {\"quality\": 0.1876068376068376, \"cost\": 0.005893562, \"time\": 85.50352709293365}, \"27ba0964b2\": {\"quality\": 0.18205128205128207, \"cost\": 0.0012502439999999997, \"time\": 50.85352149009705}, \"27daa50458\": {\"quality\": 0.21752136752136753, \"cost\": 0.005911454, \"time\": 49.84150557518005}, \"28369b2421\": {\"quality\": 0.3175213675213675, \"cost\": 0.008581950000000001, \"time\": 82.63782026767731}, \"28421e6d62\": {\"quality\": 0.05, \"cost\": 0.0071309500000000005, \"time\": 77.62054443359375}, \"2848c42f91\": {\"quality\": 0.0, \"cost\": 0.004609089, \"time\": 92.82525777816772}, \"28a638bb6e\": {\"quality\": 0.04444444444444444, \"cost\": 0.0037765230000000004, \"time\": 103.16399810314178}, \"290947fe5a\": {\"quality\": 0.08482905982905983, \"cost\": 0.005273313, \"time\": 115.58293502330781}, \"2936c3e43e\": {\"quality\": 0.13205128205128205, \"cost\": 0.011067623, \"time\": 92.91213045120239}, \"293ec5edca\": {\"quality\": 0.2987179487179487, \"cost\": 0.006866393, \"time\": 77.98297226428986}, \"294e541235\": {\"quality\": 0.0, \"cost\": 0.0024907139999999998, \"time\": 55.42697856426239}, \"295ed5e759\": {\"quality\": 0.023076923076923078, \"cost\": 0.007770515999999999, \"time\": 73.042240858078}, \"2960431101\": {\"quality\": 0.21752136752136753, \"cost\": 0.007103678, \"time\": 72.23796293735504}, \"29892d8468\": {\"quality\": 0.0, \"cost\": 0.011991246, \"time\": 90.30782704353332}, \"299a0aeb65\": {\"quality\": 0.21752136752136753, \"cost\": 0.008716849999999998, \"time\": 87.09911065101625}, \"29bf3c0a3b\": {\"quality\": 0.0952991452991453, \"cost\": 0.008317653000000001, \"time\": 95.62355568408967}, \"29c8c693e2\": {\"quality\": 0.19252136752136753, \"cost\": 0.009049301000000001, \"time\": 97.66356644630432}, \"2a7d15f4a7\": {\"quality\": 0.1987179487179487, \"cost\": 0.0007805049999999999, \"time\": 26.81472017765045}, \"2ae24e0124\": {\"quality\": 0.2098290598290598, \"cost\": 0.006997367999999999, \"time\": 100.80345997810363}, \"2b5679d248\": {\"quality\": 0.23482905982905983, \"cost\": 0.012146391000000001, \"time\": 73.05133624076844}, \"2b5ab72a55\": {\"quality\": 0.1, \"cost\": 0.00048774399999999997, \"time\": 87.95751609802247}, \"2b82a67eb1\": {\"quality\": 0.2098290598290598, \"cost\": 0.002022681, \"time\": 73.1611754655838}, \"2bcbffdf85\": {\"quality\": 0.1, \"cost\": 0.003530234999999999, \"time\": 89.18944058418273}, \"2bd39ee744\": {\"quality\": 0.2098290598290598, \"cost\": 0.014582621000000002, \"time\": 59.03191387653351}, \"2bf38d797f\": {\"quality\": 0.2064102564102564, \"cost\": 0.0075413270000000004, \"time\": 54.856262516975406}, \"2c1640adf7\": {\"quality\": 0.18482905982905984, \"cost\": 0.0019162139999999999, \"time\": 77.58103239536285}, \"2c5cf9eb26\": {\"quality\": 0.125, \"cost\": 0.006593481, \"time\": 71.76690304279327}, \"2c87313a93\": {\"quality\": 0.11752136752136753, \"cost\": 0.012109324000000001, \"time\": 136.4559998512268}, \"2c9a9f94c4\": {\"quality\": 0.06538461538461539, \"cost\": 0.011352702, \"time\": 108.1334749698639}, \"2d3bbc2d23\": {\"quality\": 0.1987179487179487, \"cost\": 0.006806840999999999, \"time\": 94.72236230373383}, \"2d7f1dbd4b\": {\"quality\": 0.1987179487179487, \"cost\": 0.003597009, \"time\": 112.8651005268097}, \"2de113167b\": {\"quality\": 0.11752136752136753, \"cost\": 0.011756008, \"time\": 101.33509397506714}, \"2de3eb2c19\": {\"quality\": 0.23974358974358972, \"cost\": 0.006698560000000001, \"time\": 58.83604803085328}, \"2e02b71061\": {\"quality\": 0.09444444444444444, \"cost\": 0.002093574, \"time\": 88.67284088134765}, \"2e30394ac6\": {\"quality\": 0.07307692307692308, \"cost\": 0.007492572, \"time\": 90.685981965065}, \"2e5d071f21\": {\"quality\": 0.12863247863247865, \"cost\": 0.0072971500000000005, \"time\": 123.94845788478851}, \"2e9c5cc9bf\": {\"quality\": 0.05, \"cost\": 0.005878296000000001, \"time\": 49.976054430007935}, \"2ec4bec1a3\": {\"quality\": 0.21474358974358973, \"cost\": 0.010779144000000001, \"time\": 124.55905365943909}, \"2f1573da80\": {\"quality\": 0.1987179487179487, \"cost\": 0.008802546, \"time\": 103.0501886844635}, \"2fc0cb3592\": {\"quality\": 0.06538461538461539, \"cost\": 0.00690309, \"time\": 73.22720482349396}, \"2fd9cd426a\": {\"quality\": 0.21752136752136753, \"cost\": 0.01168472, \"time\": 97.46322917938232}, \"3019af79b3\": {\"quality\": 0.1, \"cost\": 0.001074796, \"time\": 51.414027357101446}, \"302c1d97fc\": {\"quality\": 0.1814102564102564, \"cost\": 0.01244514, \"time\": 70.44251236915588}, \"303b467574\": {\"quality\": 0.21752136752136753, \"cost\": 0.007695415999999999, \"time\": 104.79880075454712}, \"3058b1f1f8\": {\"quality\": 0.12863247863247865, \"cost\": 0.006238776, \"time\": 104.19706840515136}, \"30ae4cbe91\": {\"quality\": 0.19252136752136753, \"cost\": 0.000758614, \"time\": 64.8543310880661}, \"30e3ff1d17\": {\"quality\": 0.1841880341880342, \"cost\": 0.007638456, \"time\": 97.64294464588164}, \"30f20c8fe6\": {\"quality\": 0.1876068376068376, \"cost\": 0.006185881000000001, \"time\": 95.83135304450988}, \"316759d191\": {\"quality\": 0.06538461538461539, \"cost\": 0.01277288, \"time\": 106.51430850028991}, \"3177802176\": {\"quality\": 0.05, \"cost\": 0.000812176, \"time\": 68.97149279117585}, \"3184f977a8\": {\"quality\": 0.06944444444444445, \"cost\": 0.003731166, \"time\": 116.78110229969025}, \"3194e440cf\": {\"quality\": 0.1, \"cost\": 0.0021085889999999997, \"time\": 68.94484202861786}, \"3197ad4faf\": {\"quality\": 0.125, \"cost\": 0.0029357279999999994, \"time\": 104.7419487476349}, \"31a423a3bf\": {\"quality\": 0.0, \"cost\": 0.002416362, \"time\": 71.78247156143189}, \"321e17afbd\": {\"quality\": 0.1952991452991453, \"cost\": 0.009030642000000002, \"time\": 102.16506278514862}, \"32b101d807\": {\"quality\": 0.21752136752136753, \"cost\": 0.013004699000000002, \"time\": 107.63277230262756}, \"332a350ea2\": {\"quality\": 0.10363247863247865, \"cost\": 0.005886114, \"time\": 79.25630362033844}, \"33459cd29c\": {\"quality\": 0.02222222222222222, \"cost\": 0.009792765999999998, \"time\": 109.5698808670044}, \"33a187e74f\": {\"quality\": 0.1987179487179487, \"cost\": 0.0057371969999999994, \"time\": 108.5205159664154}, \"34026bb5cc\": {\"quality\": 0.2098290598290598, \"cost\": 0.003850629, \"time\": 79.99503300189971}, \"3511b5e1d0\": {\"quality\": 0.0, \"cost\": 0.007917534, \"time\": 106.0169960975647}, \"3513e54767\": {\"quality\": 0.12307692307692308, \"cost\": 0.00652464, \"time\": 56.468523144721985}, \"353f0cb1ac\": {\"quality\": 0.2064102564102564, \"cost\": 0.0027354140000000002, \"time\": 50.53772373199463}, \"3550bf88cb\": {\"quality\": 0.22863247863247865, \"cost\": 0.007747509, \"time\": 104.70380001068116}, \"35610fb420\": {\"quality\": 0.2098290598290598, \"cost\": 0.004438119, \"time\": 84.96952936649322}, \"357267e14b\": {\"quality\": 0.17222222222222222, \"cost\": 0.00107605, \"time\": 39.50561866760254}, \"36011c7606\": {\"quality\": 0.1987179487179487, \"cost\": 0.008097354, \"time\": 84.37511310577392}, \"362d480d6d\": {\"quality\": 0.22863247863247865, \"cost\": 0.010916724, \"time\": 110.61522128582001}, \"363209b6e7\": {\"quality\": 0.32863247863247863, \"cost\": 0.006868834000000001, \"time\": 83.50905873775483}, \"3637084f91\": {\"quality\": 0.3175213675213675, \"cost\": 0.007119692, \"time\": 82.10715711116791}, \"36b17c40f3\": {\"quality\": 0.03418803418803419, \"cost\": 0.004674881999999999, \"time\": 53.087517738342285}, \"36c66671ee\": {\"quality\": 0.12307692307692308, \"cost\": 0.006580928999999999, \"time\": 86.52281455993652}, \"372e8b5f4f\": {\"quality\": 0.058333333333333334, \"cost\": 0.001610232, \"time\": 70.54334411621093}, \"375ed248fe\": {\"quality\": 0.06752136752136753, \"cost\": 0.008873826000000001, \"time\": 86.2198546409607}, \"377b8b0bcc\": {\"quality\": 0.025, \"cost\": 0.0049981109999999995, \"time\": 106.38127601146698}, \"37bd28f2c9\": {\"quality\": 0.21752136752136753, \"cost\": 0.007961847, \"time\": 103.88883001804352}, \"38075bb01f\": {\"quality\": 0.21666666666666667, \"cost\": 0.003875973, \"time\": 75.78762099742889}, \"38567d6a43\": {\"quality\": 0.0, \"cost\": 0.004616319, \"time\": 89.08885374069214}, \"389a99ab21\": {\"quality\": 0.07371794871794872, \"cost\": 0.007963053, \"time\": 102.66287496089936}, \"389c54cbca\": {\"quality\": 0.0702991452991453, \"cost\": 0.005610244, \"time\": 48.15491397380829}, \"38ec11cf7b\": {\"quality\": 0.0, \"cost\": 0.007187318999999999, \"time\": 103.98532650470733}, \"3980f20caa\": {\"quality\": 0.04807692307692308, \"cost\": 0.014391898, \"time\": 100.08462851047516}, \"39cd4ca402\": {\"quality\": 0.025, \"cost\": 0.003120591, \"time\": 75.18385965824127}, \"3a34b24c41\": {\"quality\": 0.21752136752136753, \"cost\": 0.002965395, \"time\": 73.19122822284699}, \"3b2e8075ea\": {\"quality\": 0.2098290598290598, \"cost\": 0.003075393, \"time\": 70.68897068500519}, \"3b3676521a\": {\"quality\": 0.21752136752136753, \"cost\": 0.013715839000000002, \"time\": 107.15725564956665}, \"3b3a6bf087\": {\"quality\": 0.30982905982905984, \"cost\": 0.006470724000000001, \"time\": 74.07467935085296}, \"3b4bde0121\": {\"quality\": 0.0, \"cost\": 0.00302925, \"time\": 77.53029568195343}, \"3b57530a56\": {\"quality\": 0.2098290598290598, \"cost\": 0.002663507, \"time\": 48.332898473739625}, \"3c206c89f3\": {\"quality\": 0.21752136752136753, \"cost\": 0.008676515999999999, \"time\": 115.085169839859}, \"3cbab8082e\": {\"quality\": 0.09871794871794873, \"cost\": 0.008985135, \"time\": 116.89071977138519}, \"3d71c4dd2c\": {\"quality\": 0.10982905982905983, \"cost\": 0.006464912999999999, \"time\": 86.58996284008026}, \"3ea15ac20c\": {\"quality\": 0.11538461538461539, \"cost\": 0.007241556, \"time\": 125.68785231113434}, \"3f1a58aec9\": {\"quality\": 0.22863247863247865, \"cost\": 0.00189393, \"time\": 45.61551144123077}, \"3f2321bb08\": {\"quality\": 0.125, \"cost\": 0.000410382, \"time\": 33.271996068954465}, \"3f3ef494b0\": {\"quality\": 0.0, \"cost\": 0.001176912, \"time\": 50.03896124362946}, \"3f62c3fbfc\": {\"quality\": 0.21752136752136753, \"cost\": 0.004284207, \"time\": 95.22124736309053}, \"3f88dd99f7\": {\"quality\": 0.19252136752136753, \"cost\": 0.005681200000000001, \"time\": 63.5021466255188}, \"3fa747af9a\": {\"quality\": 0.15833333333333333, \"cost\": 0.002231574, \"time\": 96.99003422260284}, \"40104c813f\": {\"quality\": 0.0, \"cost\": 0.010208657000000001, \"time\": 125.99128649234771}, \"403b05da2d\": {\"quality\": 0.08141025641025641, \"cost\": 0.010538071999999999, \"time\": 94.6719701051712}, \"4043815a3e\": {\"quality\": 0.21752136752136753, \"cost\": 0.013145456, \"time\": 124.91901173591614}, \"409ff67607\": {\"quality\": 0.06944444444444445, \"cost\": 0.004053711, \"time\": 99.38009803295135}, \"412c065b83\": {\"quality\": 0.0, \"cost\": 0.008236578, \"time\": 92.44097802639007}, \"4171fbac5c\": {\"quality\": 0.19444444444444442, \"cost\": 0.009067686, \"time\": 114.00634713172913}, \"4191118787\": {\"quality\": 0.1987179487179487, \"cost\": 0.008734028999999999, \"time\": 109.63090167045593}, \"41d5b97871\": {\"quality\": 0.0, \"cost\": 0.002514906, \"time\": 62.46952087879181}, \"41d8845655\": {\"quality\": 0.2098290598290598, \"cost\": 0.010189181000000002, \"time\": 59.657727408409116}, \"41ee202cac\": {\"quality\": 0.06538461538461539, \"cost\": 0.000921442, \"time\": 54.925141382217404}, \"42082dcd0d\": {\"quality\": 0.1987179487179487, \"cost\": 0.007583567999999999, \"time\": 108.27059605121613}, \"42ddd48341\": {\"quality\": 0.22094017094017093, \"cost\": 0.014396616, \"time\": 104.56641755104064}, \"4361bc7ea7\": {\"quality\": 0.13333333333333333, \"cost\": 0.001473496, \"time\": 63.439108324050906}, \"43afdad250\": {\"quality\": 0.22863247863247865, \"cost\": 0.011869643, \"time\": 108.28341348171234}, \"43c3cf9cb8\": {\"quality\": 0.2064102564102564, \"cost\": 0.0068331120000000006, \"time\": 53.15135598182678}, \"43d24fb32a\": {\"quality\": 0.07307692307692308, \"cost\": 0.00976587, \"time\": 85.51874697208405}, \"440bc872de\": {\"quality\": 0.07222222222222222, \"cost\": 0.007188219, \"time\": 121.43468098640442}, \"44173a9aef\": {\"quality\": 0.30641025641025643, \"cost\": 0.01244934, \"time\": 115.21545617580415}, \"44d6af5523\": {\"quality\": 0.2098290598290598, \"cost\": 0.00476793, \"time\": 81.34744908809662}, \"44fe4e4e3e\": {\"quality\": 0.11752136752136753, \"cost\": 0.01136529, \"time\": 81.14654626846314}, \"4587a1500c\": {\"quality\": 0.3175213675213675, \"cost\": 0.006090974000000002, \"time\": 79.77526035308838}, \"45ef93b61e\": {\"quality\": 0.1987179487179487, \"cost\": 0.003150051, \"time\": 81.59605078697206}, \"461846a52d\": {\"quality\": 0.1814102564102564, \"cost\": 0.003777498, \"time\": 82.82821555137635}, \"462e6ff849\": {\"quality\": 0.11752136752136753, \"cost\": 0.010684958, \"time\": 113.33498125076294}, \"4630853d32\": {\"quality\": 0.06752136752136753, \"cost\": 0.008278823000000001, \"time\": 86.25940473079682}, \"46475b9e75\": {\"quality\": 0.2098290598290598, \"cost\": 0.0061740449999999995, \"time\": 87.75788187980652}, \"46654a1f32\": {\"quality\": 0.0, \"cost\": 0.011978384, \"time\": 67.87502360343933}, \"466a3036b2\": {\"quality\": 0.09252136752136753, \"cost\": 0.010652967, \"time\": 114.7115253686905}, \"466d4d16dd\": {\"quality\": 0.18482905982905984, \"cost\": 0.008396510000000001, \"time\": 86.67418849468231}, \"46a35022d8\": {\"quality\": 0.1952991452991453, \"cost\": 0.000766365, \"time\": 59.83990566730499}, \"46ed68152d\": {\"quality\": 0.18333333333333332, \"cost\": 0.01143953, \"time\": 92.87648718357087}, \"46edc488a4\": {\"quality\": 0.059829059829059825, \"cost\": 0.0012849899999999997, \"time\": 50.55697724819183}, \"476a12876c\": {\"quality\": 0.11752136752136753, \"cost\": 0.006154817999999999, \"time\": 81.70997877120972}, \"48043e2304\": {\"quality\": 0.2098290598290598, \"cost\": 0.009217511000000001, \"time\": 89.0722332715988}, \"488645cbd9\": {\"quality\": 0.2064102564102564, \"cost\": 0.001493856, \"time\": 48.69374096393585}, \"49009a3b57\": {\"quality\": 0.059829059829059825, \"cost\": 0.007598573999999999, \"time\": 78.47919590473174}, \"4909061216\": {\"quality\": 0.21752136752136753, \"cost\": 0.013011773, \"time\": 104.75487668514252}, \"49107972df\": {\"quality\": 0.025, \"cost\": 0.007639823999999999, \"time\": 118.11295173168182}, \"49731b1ccd\": {\"quality\": 0.0952991452991453, \"cost\": 0.006953804999999999, \"time\": 77.60816009044646}, \"498f146004\": {\"quality\": 0.11752136752136753, \"cost\": 0.008050870000000002, \"time\": 112.59319038391114}, \"49ad844bd2\": {\"quality\": 0.2098290598290598, \"cost\": 0.008512473, \"time\": 109.38159234523773}, \"49ca727e49\": {\"quality\": 0.2098290598290598, \"cost\": 0.000746866, \"time\": 34.221199584007266}, \"4a4a960a82\": {\"quality\": 0.13333333333333333, \"cost\": 0.007445242000000001, \"time\": 84.403648686409}, \"4a92372986\": {\"quality\": 0.16538461538461538, \"cost\": 0.00319335, \"time\": 83.45380291938781}, \"4aa7e8fde6\": {\"quality\": 0.10641025641025642, \"cost\": 0.0019865879999999996, \"time\": 83.08999395370483}, \"4aafd39d76\": {\"quality\": 0.14038461538461539, \"cost\": 0.005373513, \"time\": 82.9023279428482}, \"4ace1cfad1\": {\"quality\": 0.16538461538461538, \"cost\": 0.009508553999999999, \"time\": 106.34385075569153}, \"4ad1952206\": {\"quality\": 0.20213675213675214, \"cost\": 0.0085493, \"time\": 83.97655324935913}, \"4b59f40131\": {\"quality\": 0.04807692307692308, \"cost\": 0.006207440999999999, \"time\": 83.43397681713105}, \"4b86a1c038\": {\"quality\": 0.2064102564102564, \"cost\": 0.00792555, \"time\": 106.1481963634491}, \"4b92a26754\": {\"quality\": 0.0, \"cost\": 0.00047688399999999996, \"time\": 69.90886387825012}, \"4bc4528402\": {\"quality\": 0.16538461538461538, \"cost\": 0.004527411, \"time\": 81.34518160820008}, \"4c158a1a4a\": {\"quality\": 0.08205128205128205, \"cost\": 0.007535195999999999, \"time\": 78.38852503299714}, \"4c954323e3\": {\"quality\": 0.21752136752136753, \"cost\": 0.00974773, \"time\": 103.17564594745636}, \"4d8bcf8ae2\": {\"quality\": 0.125, \"cost\": 0.000539128, \"time\": 69.75875072479249}, \"4d91e8a27b\": {\"quality\": 0.21752136752136753, \"cost\": 0.010584353000000001, \"time\": 76.16667878627777}, \"4dd3635bc3\": {\"quality\": 0.2098290598290598, \"cost\": 0.012259316000000001, \"time\": 78.35238783359529}, \"4dd96bd18f\": {\"quality\": 0.21752136752136753, \"cost\": 0.011986314000000001, \"time\": 105.92028946876526}, \"4e298ee0d4\": {\"quality\": 0.1814102564102564, \"cost\": 0.008259033, \"time\": 95.37437601089476}, \"4e3443a0f9\": {\"quality\": 0.11752136752136753, \"cost\": 0.013054975, \"time\": 106.09084448814392}, \"4e4b9db2b8\": {\"quality\": 0.21752136752136753, \"cost\": 0.007140224000000001, \"time\": 72.97610201835633}, \"4e6509f614\": {\"quality\": 0.061111111111111116, \"cost\": 0.0053766, \"time\": 47.53030514717102}, \"4e6a83e751\": {\"quality\": 0.07371794871794872, \"cost\": 0.001089142, \"time\": 53.57680480480194}, \"4e8d8e527a\": {\"quality\": 0.14871794871794872, \"cost\": 0.0051247079999999995, \"time\": 85.9967652797699}, \"4ef333ab21\": {\"quality\": 0.19444444444444442, \"cost\": 0.001917522, \"time\": 79.47004499435425}, \"4f16545711\": {\"quality\": 0.06666666666666667, \"cost\": 0.005466131999999999, \"time\": 88.67296268939972}, \"4f8cca1195\": {\"quality\": 0.21752136752136753, \"cost\": 0.010168095, \"time\": 114.15254590511321}, \"500860eaa2\": {\"quality\": 0.0, \"cost\": 0.0022933619999999997, \"time\": 38.243334197998045}, \"50701b505e\": {\"quality\": 0.1987179487179487, \"cost\": 0.0064327049999999995, \"time\": 79.18819501399994}, \"50bc87e9cc\": {\"quality\": 0.1987179487179487, \"cost\": 0.005166006000000001, \"time\": 86.32211146354675}, \"50c03be77c\": {\"quality\": 0.10982905982905983, \"cost\": 0.00168543, \"time\": 94.39419853687286}, \"510375edad\": {\"quality\": 0.0, \"cost\": 0.0021525, \"time\": 67.02195911407472}, \"512fdb607c\": {\"quality\": 0.2098290598290598, \"cost\": 0.008402651, \"time\": 120.24980294704437}, \"51583a901c\": {\"quality\": 0.2098290598290598, \"cost\": 0.010226283000000001, \"time\": 109.70203943252564}, \"521314dab6\": {\"quality\": 0.19252136752136753, \"cost\": 0.0064746199999999995, \"time\": 59.58354845046997}, \"5241bf401b\": {\"quality\": 0.07222222222222222, \"cost\": 0.006321017999999999, \"time\": 82.37689163684846}, \"526878b5eb\": {\"quality\": 0.07371794871794872, \"cost\": 0.008675337, \"time\": 115.97295072078705}, \"52c1cba6ce\": {\"quality\": 0.21752136752136753, \"cost\": 0.009671612999999999, \"time\": 108.82956576347351}, \"52e5d0f4fb\": {\"quality\": 0.2064102564102564, \"cost\": 0.001697778, \"time\": 88.22295272350311}, \"52f041a70e\": {\"quality\": 0.2098290598290598, \"cost\": 0.001067476, \"time\": 32.61202094554901}, \"5307496302\": {\"quality\": 0.2098290598290598, \"cost\": 0.005388657, \"time\": 86.62635931968688}, \"53869388bb\": {\"quality\": 0.061111111111111116, \"cost\": 0.0021395499999999996, \"time\": 39.578048658370975}, \"53d2932c4f\": {\"quality\": 0.2814102564102564, \"cost\": 0.007557482000000001, \"time\": 73.93080537319183}, \"5474247f91\": {\"quality\": 0.0, \"cost\": 0.008248308, \"time\": 80.75084838867187}, \"557d2cf7ba\": {\"quality\": 0.19444444444444442, \"cost\": 0.009174846, \"time\": 81.92483322620392}, \"559c7120c5\": {\"quality\": 0.14038461538461539, \"cost\": 0.009353153, \"time\": 113.30842261314393}, \"55c8aa8935\": {\"quality\": 0.04807692307692308, \"cost\": 0.01165586, \"time\": 114.18454928398131}, \"56a29a28c5\": {\"quality\": 0.04807692307692308, \"cost\": 0.004978499999999999, \"time\": 55.467578983306886}, \"56b39eb1d6\": {\"quality\": 0.21752136752136753, \"cost\": 0.009558101, \"time\": 131.33570635318756}, \"5703697dbd\": {\"quality\": 0.2098290598290598, \"cost\": 0.006108023999999999, \"time\": 79.37135038375854}, \"5718f2ed80\": {\"quality\": 0.10641025641025642, \"cost\": 0.003117972, \"time\": 91.63201060295106}, \"572a02a59a\": {\"quality\": 0.19444444444444442, \"cost\": 0.004321886999999999, \"time\": 100.45879077911377}, \"572c2df793\": {\"quality\": 0.04038461538461539, \"cost\": 0.004394319000000001, \"time\": 143.467453289032}, \"5750713a41\": {\"quality\": 0.2098290598290598, \"cost\": 0.006046722, \"time\": 92.31352968215941}, \"57757ef15e\": {\"quality\": 0.1876068376068376, \"cost\": 0.006326642, \"time\": 106.06430275440215}, \"5793d14bbe\": {\"quality\": 0.2098290598290598, \"cost\": 0.009510144000000002, \"time\": 139.83697164058685}, \"579a915ed2\": {\"quality\": 0.08333333333333334, \"cost\": 0.011355894000000002, \"time\": 109.32498943805695}, \"579c81bbe0\": {\"quality\": 0.025, \"cost\": 0.004824098, \"time\": 75.78087794780731}, \"57bed1722f\": {\"quality\": 0.025, \"cost\": 0.005056998, \"time\": 85.83169932365418}, \"585ba6d20b\": {\"quality\": 0.21752136752136753, \"cost\": 0.007866175, \"time\": 151.51093373298644}, \"589a1cea79\": {\"quality\": 0.05, \"cost\": 0.009551150000000001, \"time\": 151.7195835828781}, \"59006532b4\": {\"quality\": 0.16538461538461538, \"cost\": 0.007208474999999999, \"time\": 118.06772444248199}, \"59326c4e00\": {\"quality\": 0.015384615384615385, \"cost\": 0.005300058, \"time\": 72.70066883563996}, \"596f0ed542\": {\"quality\": 0.023076923076923078, \"cost\": 0.011558136, \"time\": 137.05188086032868}, \"5971ba4e0d\": {\"quality\": 0.21752136752136753, \"cost\": 0.0018305540000000003, \"time\": 43.38757050037384}, \"59e0117b7d\": {\"quality\": 0.21752136752136753, \"cost\": 0.006857488, \"time\": 139.82263526916503}, \"59f515d0da\": {\"quality\": 0.2064102564102564, \"cost\": 0.00388749, \"time\": 108.1351862192154}, \"59f887b67c\": {\"quality\": 0.0, \"cost\": 0.010208398, \"time\": 146.34244396686552}, \"5a22920db4\": {\"quality\": 0.1987179487179487, \"cost\": 0.0018006569999999998, \"time\": 75.78275735378266}, \"5a35020d45\": {\"quality\": 0.1987179487179487, \"cost\": 0.0060887910000000005, \"time\": 109.82949197292328}, \"5aa71bb88a\": {\"quality\": 0.12094017094017094, \"cost\": 0.000638436, \"time\": 67.15015056133271}, \"5b10fbdbe1\": {\"quality\": 0.21752136752136753, \"cost\": 0.013089312000000002, \"time\": 128.6064488887787}, \"5b4ad39a9e\": {\"quality\": 0.11752136752136753, \"cost\": 0.0032681159999999997, \"time\": 99.70189120769501}, \"5bade9eb85\": {\"quality\": 0.18482905982905984, \"cost\": 0.008712719999999998, \"time\": 120.3732186794281}, \"5be16744bf\": {\"quality\": 0.1952991452991453, \"cost\": 0.013749790000000001, \"time\": 91.71922521591186}, \"5c0db11303\": {\"quality\": 0.058333333333333334, \"cost\": 0.002397702, \"time\": 93.91862483024596}, \"5c53feccd9\": {\"quality\": 0.023076923076923078, \"cost\": 0.015926556, \"time\": 118.78805196285248}, \"5c77c7c2b2\": {\"quality\": 0.15, \"cost\": 0.008434941000000001, \"time\": 85.08626408576964}, \"5d072194b8\": {\"quality\": 0.11752136752136753, \"cost\": 0.007648864000000002, \"time\": 104.28334321975709}, \"5d79b50feb\": {\"quality\": 0.14038461538461539, \"cost\": 0.013589224, \"time\": 84.18065688610076}, \"5dc216cd6b\": {\"quality\": 0.10982905982905983, \"cost\": 0.010508788, \"time\": 83.96197824478149}, \"5dd68c1b8f\": {\"quality\": 0.16944444444444445, \"cost\": 0.001444968, \"time\": 54.59350550174713}, \"5de4a882c1\": {\"quality\": 0.21752136752136753, \"cost\": 0.008734155, \"time\": 75.31518497467042}, \"5deeeb223f\": {\"quality\": 0.24316239316239316, \"cost\": 0.005724273, \"time\": 115.48380098342895}, \"5e2f03b962\": {\"quality\": 0.1876068376068376, \"cost\": 0.0007302779999999999, \"time\": 48.90210340023041}, \"5ea2fab380\": {\"quality\": 0.05149572649572649, \"cost\": 0.0048052319999999996, \"time\": 47.63824257850647}, \"5eb3bb525b\": {\"quality\": 0.0702991452991453, \"cost\": 0.008967890000000001, \"time\": 96.21212074756622}, \"5ec3832817\": {\"quality\": 0.0, \"cost\": 0.008163805, \"time\": 148.1108601331711}, \"5f0199e07b\": {\"quality\": 0.025, \"cost\": 0.00068568, \"time\": 49.921340346336365}, \"5f37b3902b\": {\"quality\": 0.04038461538461539, \"cost\": 0.007223498, \"time\": 77.24008927345275}, \"60cb623c53\": {\"quality\": 0.04871794871794872, \"cost\": 0.002396232, \"time\": 54.30748798847199}, \"612e546d71\": {\"quality\": 0.10705128205128206, \"cost\": 0.008148234, \"time\": 151.1273174762726}, \"6160bfb439\": {\"quality\": 0.11474358974358972, \"cost\": 0.010103339999999999, \"time\": 148.44622106552123}, \"6178f33808\": {\"quality\": 0.04722222222222222, \"cost\": 0.006706226999999999, \"time\": 153.97144429683686}, \"619b48dde9\": {\"quality\": 0.21752136752136753, \"cost\": 0.006427425999999999, \"time\": 120.5339899301529}, \"6268ac658c\": {\"quality\": 0.24316239316239316, \"cost\": 0.01069585, \"time\": 144.7754723072052}, \"628f34aace\": {\"quality\": 0.2098290598290598, \"cost\": 0.005102127, \"time\": 151.19322457313538}, \"630d1ecda0\": {\"quality\": 0.0, \"cost\": 0.00503412, \"time\": 158.05562288761138}, \"63a0aaebed\": {\"quality\": 0.15833333333333333, \"cost\": 0.001865547, \"time\": 88.02101895809173}, \"63f392465f\": {\"quality\": 0.18333333333333332, \"cost\": 0.0020362590000000003, \"time\": 114.81161072254181}, \"6527f214c3\": {\"quality\": 0.125, \"cost\": 0.0020534459999999996, \"time\": 107.1269181728363}, \"652c0f4bdf\": {\"quality\": 0.1, \"cost\": 0.013503214, \"time\": 140.5670464038849}, \"6533c85913\": {\"quality\": 0.10641025641025642, \"cost\": 0.009929774999999998, \"time\": 122.77552180290222}, \"65627426e0\": {\"quality\": 0.15, \"cost\": 0.000700312, \"time\": 55.69904806613922}, \"65801893b4\": {\"quality\": 0.025, \"cost\": 0.01106166, \"time\": 113.8573011636734}, \"65b76da9c6\": {\"quality\": 0.12222222222222223, \"cost\": 0.007336156, \"time\": 88.77893948554993}, \"65be1c1306\": {\"quality\": 0.06752136752136753, \"cost\": 0.001988206, \"time\": 34.18720245361328}, \"65e0216208\": {\"quality\": 0.05, \"cost\": 0.008013333, \"time\": 111.02614188194275}, \"65eee615d7\": {\"quality\": 0.1987179487179487, \"cost\": 0.000519715, \"time\": 32.74071106910706}, \"66277da52f\": {\"quality\": 0.09871794871794873, \"cost\": 0.0036039389999999996, \"time\": 110.55973320007324}, \"66750c0934\": {\"quality\": 0.21752136752136753, \"cost\": 0.007508133, \"time\": 85.75274183750153}, \"66776ec181\": {\"quality\": 0.10641025641025642, \"cost\": 0.006989541, \"time\": 111.77394149303436}, \"67632141f6\": {\"quality\": 0.04038461538461539, \"cost\": 0.004542882, \"time\": 63.92582683563233}, \"677deb302a\": {\"quality\": 0.1814102564102564, \"cost\": 0.007409720999999999, \"time\": 111.17820315361024}, \"67868fcff6\": {\"quality\": 0.21752136752136753, \"cost\": 0.011676892000000001, \"time\": 81.84591567516327}, \"67aad9ea16\": {\"quality\": 0.2098290598290598, \"cost\": 0.007452915, \"time\": 126.7279718399048}, \"67bab6732d\": {\"quality\": 0.10555555555555557, \"cost\": 0.01121626, \"time\": 84.01423738002777}, \"67fe399cf1\": {\"quality\": 0.21752136752136753, \"cost\": 0.007087558, \"time\": 80.51065149307252}, \"6846bd8fb3\": {\"quality\": 0.08141025641025641, \"cost\": 0.006947139, \"time\": 85.74786493778228}, \"68583552fb\": {\"quality\": 0.0, \"cost\": 0.0053533439999999995, \"time\": 93.39369978904725}, \"689e327daf\": {\"quality\": 0.08333333333333334, \"cost\": 0.000799428, \"time\": 57.74340398311615}, \"69b3b67de6\": {\"quality\": 0.23974358974358972, \"cost\": 0.008200215, \"time\": 95.98227195739746}, \"69bf3f6ba0\": {\"quality\": 0.21752136752136753, \"cost\": 0.008827848, \"time\": 102.32003858089448}, \"69f90e610f\": {\"quality\": 0.22585470085470086, \"cost\": 0.011046368, \"time\": 121.27799794673919}, \"6a022c3f73\": {\"quality\": 0.21944444444444444, \"cost\": 0.004195854, \"time\": 99.68456645011902}, \"6a10c53ad8\": {\"quality\": 0.32863247863247863, \"cost\": 0.012164633000000001, \"time\": 119.7128321170807}, \"6a6348f69d\": {\"quality\": 0.025, \"cost\": 0.0010719339999999999, \"time\": 79.13316841125489}, \"6a8726145c\": {\"quality\": 0.1876068376068376, \"cost\": 0.0014679629999999996, \"time\": 77.99199786186219}, \"6a8a675442\": {\"quality\": 0.22094017094017093, \"cost\": 0.005341439999999999, \"time\": 90.32394058704377}, \"6aac59742a\": {\"quality\": 0.0, \"cost\": 0.009132285, \"time\": 137.44288988113402}, \"6ac193c88f\": {\"quality\": 0.25085470085470085, \"cost\": 0.006668448, \"time\": 121.86276133060456}, \"6ae9e9de0b\": {\"quality\": 0.21752136752136753, \"cost\": 0.0077003290000000005, \"time\": 125.54181215763091}, \"6b0c585f5c\": {\"quality\": 0.21752136752136753, \"cost\": 0.006663018, \"time\": 122.58973615169526}, \"6b3c16def2\": {\"quality\": 0.18482905982905984, \"cost\": 0.001533927, \"time\": 51.53218412399292}, \"6c05c47050\": {\"quality\": 0.1987179487179487, \"cost\": 0.003248111999999999, \"time\": 96.47181532382965}, \"6c3667811b\": {\"quality\": 0.21474358974358973, \"cost\": 0.005244696, \"time\": 89.6854020357132}, \"6cc813aa68\": {\"quality\": 0.14807692307692308, \"cost\": 0.005938046000000001, \"time\": 58.51538195610046}, \"6cd78cac7e\": {\"quality\": 0.09444444444444444, \"cost\": 0.007387408, \"time\": 101.36276533603669}, \"6d20c6ace0\": {\"quality\": 0.21752136752136753, \"cost\": 0.024144216, \"time\": 121.96203644275664}, \"6d67c56ba6\": {\"quality\": 0.17307692307692307, \"cost\": 0.009393948, \"time\": 150.0689915418625}, \"6db70dc3b6\": {\"quality\": 0.04038461538461539, \"cost\": 0.0015085300000000001, \"time\": 94.44009244441986}, \"6e0690f576\": {\"quality\": 0.0, \"cost\": 0.001440486, \"time\": 74.4156935930252}, \"6e3db7ec5e\": {\"quality\": 0.21752136752136753, \"cost\": 0.009058176, \"time\": 141.13843698501586}, \"6e62bbb47f\": {\"quality\": 0.15982905982905982, \"cost\": 0.0032870549999999997, \"time\": 139.53261284828187}, \"6e859bfae6\": {\"quality\": 0.05, \"cost\": 0.004340145, \"time\": 162.00480234622955}, \"6e93514f45\": {\"quality\": 0.0, \"cost\": 0.0047780819999999995, \"time\": 95.28718383312224}, \"6eae47102b\": {\"quality\": 0.21752136752136753, \"cost\": 0.008071342, \"time\": 93.64182848930359}, \"6ecf93c479\": {\"quality\": 0.17307692307692307, \"cost\": 0.0028209449999999996, \"time\": 103.17148315906525}, \"6ef3b7127e\": {\"quality\": 0.21752136752136753, \"cost\": 0.006768256, \"time\": 68.10584411621093}, \"6f323f80c7\": {\"quality\": 0.0, \"cost\": 0.00037804, \"time\": 104.40542347431182}, \"6f60a05c33\": {\"quality\": 0.15833333333333333, \"cost\": 0.0037825470000000003, \"time\": 92.0291631937027}, \"6fbdd8b57c\": {\"quality\": 0.1, \"cost\": 0.002123712, \"time\": 95.29761900901795}, \"6fd6046c4b\": {\"quality\": 0.18333333333333332, \"cost\": 0.005121717, \"time\": 132.3336772441864}, \"6fe0b3f929\": {\"quality\": 0.0, \"cost\": 0.006223385999999999, \"time\": 68.93117754459381}, \"6ff4f667f8\": {\"quality\": 0.19252136752136753, \"cost\": 0.006652730000000001, \"time\": 87.94405431747437}, \"700474dfbd\": {\"quality\": 0.15833333333333333, \"cost\": 0.0028382069999999997, \"time\": 102.84161474704743}, \"700ab1d309\": {\"quality\": 0.023076923076923078, \"cost\": 0.017122988, \"time\": 117.11154954433441}, \"7040e83d52\": {\"quality\": 0.24166666666666664, \"cost\": 0.019616124999999998, \"time\": 101.78968648910522}, \"704209377f\": {\"quality\": 0.0876068376068376, \"cost\": 0.005003547, \"time\": 138.1696399450302}, \"70b666e371\": {\"quality\": 0.18482905982905984, \"cost\": 0.004856766, \"time\": 104.8691722393036}, \"70c850e039\": {\"quality\": 0.18333333333333332, \"cost\": 0.002363835, \"time\": 67.29307141304017}, \"7112a7e64c\": {\"quality\": 0.25085470085470085, \"cost\": 0.0058677550000000005, \"time\": 63.623961353302}, \"7114013f0c\": {\"quality\": 0.16538461538461538, \"cost\": 0.0022517069999999995, \"time\": 125.79093027114868}, \"715070d0ca\": {\"quality\": 0.025, \"cost\": 0.006732683999999999, \"time\": 135.16606471538546}, \"71b615468b\": {\"quality\": 0.09252136752136753, \"cost\": 0.010796074, \"time\": 94.77488939762115}, \"71ed893462\": {\"quality\": 0.19444444444444442, \"cost\": 0.011514972000000002, \"time\": 137.31028594970704}, \"722d41b2f8\": {\"quality\": 0.125, \"cost\": 0.006919008000000001, \"time\": 110.66710319519044}, \"723fd5589a\": {\"quality\": 0.125, \"cost\": 0.012675105000000002, \"time\": 105.89904611110688}, \"7250da0f41\": {\"quality\": 0.0, \"cost\": 0.0072467339999999995, \"time\": 74.25018970966339}, \"7260a96349\": {\"quality\": 0.10641025641025642, \"cost\": 0.006264144, \"time\": 160.2630994796753}, \"7347cf0308\": {\"quality\": 0.0, \"cost\": 0.003343845, \"time\": 72.2089759349823}, \"736e652158\": {\"quality\": 0.0, \"cost\": 0.005920191, \"time\": 146.5679278612137}, \"738364d6a2\": {\"quality\": 0.0, \"cost\": 0.009448194, \"time\": 159.01038644313812}, \"739b1f81dc\": {\"quality\": 0.1987179487179487, \"cost\": 0.0017973389999999998, \"time\": 73.72460424900055}, \"73c6240c29\": {\"quality\": 0.1, \"cost\": 0.005322242999999999, \"time\": 159.0038095474243}, \"73fc2767b9\": {\"quality\": 0.2064102564102564, \"cost\": 0.0037065989999999997, \"time\": 126.9765938282013}, \"7445d99939\": {\"quality\": 0.1814102564102564, \"cost\": 0.007021181999999999, \"time\": 111.21800615787507}, \"7466a5f424\": {\"quality\": 0.3175213675213675, \"cost\": 0.015386124, \"time\": 150.73878495693208}, \"74cc4b1bc4\": {\"quality\": 0.04038461538461539, \"cost\": 0.006254895, \"time\": 117.36351308822631}, \"74d7f64b8c\": {\"quality\": 0.0, \"cost\": 0.006549831000000001, \"time\": 151.8392071723938}, \"751869cbec\": {\"quality\": 0.015384615384615385, \"cost\": 0.007969879, \"time\": 134.38790624141694}, \"7524905580\": {\"quality\": 0.2098290598290598, \"cost\": 0.004221619, \"time\": 93.99037828445435}, \"752d9649f2\": {\"quality\": 0.25085470085470085, \"cost\": 0.01328176, \"time\": 121.08108768463134}, \"7558c9722d\": {\"quality\": 0.2064102564102564, \"cost\": 0.008134605999999999, \"time\": 120.57110471725464}, \"75ca9cd4f8\": {\"quality\": 0.1876068376068376, \"cost\": 0.0004974599999999999, \"time\": 68.25030062198638}, \"7604c0aa13\": {\"quality\": 0.015384615384615385, \"cost\": 0.010546463999999998, \"time\": 90.40108380317687}, \"765dbc6ad5\": {\"quality\": 0.21752136752136753, \"cost\": 0.003435049, \"time\": 94.19224355220794}, \"7707e6e7e3\": {\"quality\": 0.11474358974358972, \"cost\": 0.010274625999999999, \"time\": 94.13086493015288}, \"7765576286\": {\"quality\": 0.2098290598290598, \"cost\": 0.008545683, \"time\": 127.77431318759919}, \"77983b6105\": {\"quality\": 0.0, \"cost\": 0.002284218, \"time\": 68.26882412433625}, \"77b5740025\": {\"quality\": 0.015384615384615385, \"cost\": 0.006498776, \"time\": 106.45389134883881}, \"77c02b00c1\": {\"quality\": 0.257051282051282, \"cost\": 0.010423473999999999, \"time\": 126.61633729934692}, \"77c6a9703a\": {\"quality\": 0.2098290598290598, \"cost\": 0.007362192, \"time\": 108.2735918521881}, \"77f293b737\": {\"quality\": 0.21752136752136753, \"cost\": 0.009682152999999999, \"time\": 136.6608712911606}, \"7801da66b9\": {\"quality\": 0.0, \"cost\": 0.004307142, \"time\": 92.96408789157867}, \"7862ea67cb\": {\"quality\": 0.15149572649572648, \"cost\": 0.0014619539999999997, \"time\": 74.10243089199066}, \"786e5d0af5\": {\"quality\": 0.22094017094017093, \"cost\": 0.009905354999999998, \"time\": 129.44383957386017}, \"795d119bc7\": {\"quality\": 0.0, \"cost\": 0.006895664000000001, \"time\": 81.85152049064637}, \"7989343d94\": {\"quality\": 0.11666666666666667, \"cost\": 0.0030767639999999996, \"time\": 104.29451589584352}, \"79fad58f07\": {\"quality\": 0.21752136752136753, \"cost\": 0.002502802, \"time\": 51.48524146080017}, \"7a207b42a8\": {\"quality\": 0.023076923076923078, \"cost\": 0.008630430000000001, \"time\": 131.3380735397339}, \"7a58d3472b\": {\"quality\": 0.06752136752136753, \"cost\": 0.012301829, \"time\": 96.64494693279266}, \"7a7cc658c8\": {\"quality\": 0.06944444444444445, \"cost\": 0.009218845, \"time\": 130.32710280418394}, \"7b024a2966\": {\"quality\": 0.14807692307692308, \"cost\": 0.009636592999999999, \"time\": 127.10254328250885}, \"7b6dc3702e\": {\"quality\": 0.11752136752136753, \"cost\": 0.01032963, \"time\": 98.65728087425232}, \"7b6f44618e\": {\"quality\": 0.21752136752136753, \"cost\": 0.012669284, \"time\": 119.66786665916442}, \"7b74b23910\": {\"quality\": 0.06538461538461539, \"cost\": 0.012630904, \"time\": 93.18642947673797}, \"7b9cc96081\": {\"quality\": 0.14807692307692308, \"cost\": 0.000405328, \"time\": 79.7800312757492}, \"7c45c61d8d\": {\"quality\": 0.21752136752136753, \"cost\": 0.006698508, \"time\": 122.86696994304657}, \"7c89a2b69e\": {\"quality\": 0.025, \"cost\": 0.012249355, \"time\": 127.1623036623001}, \"7d44f0959d\": {\"quality\": 0.21752136752136753, \"cost\": 0.006413139, \"time\": 93.95755279064178}, \"7d60c38c5c\": {\"quality\": 0.2098290598290598, \"cost\": 0.0017982599999999999, \"time\": 60.89626235961914}, \"7d67e14414\": {\"quality\": 0.11752136752136753, \"cost\": 0.0037651139999999995, \"time\": 123.35564484596253}, \"7daf7ff182\": {\"quality\": 0.13974358974358975, \"cost\": 0.011279536, \"time\": 93.00705258846284}, \"7e0ad1c9c1\": {\"quality\": 0.07371794871794872, \"cost\": 0.00508167, \"time\": 85.57242858409882}, \"7ed07ad40a\": {\"quality\": 0.2098290598290598, \"cost\": 0.00644937, \"time\": 84.29863061904908}, \"7fa67a7656\": {\"quality\": 0.07307692307692308, \"cost\": 0.007675500000000001, \"time\": 69.2791738986969}, \"806881adcb\": {\"quality\": 0.2064102564102564, \"cost\": 0.008654110999999999, \"time\": 86.79695003032684}, \"80a1d9c2f3\": {\"quality\": 0.15833333333333333, \"cost\": 0.005246394, \"time\": 105.42429778575897}, \"80be7df955\": {\"quality\": 0.0, \"cost\": 0.0014326019999999998, \"time\": 58.2230907201767}, \"80bf60c422\": {\"quality\": 0.16538461538461538, \"cost\": 0.001969395, \"time\": 61.00985796451569}, \"81333c7a33\": {\"quality\": 0.1987179487179487, \"cost\": 0.018448848000000004, \"time\": 86.57931089401245}, \"813e75210b\": {\"quality\": 0.23632478632478632, \"cost\": 0.002293942, \"time\": 33.89890332221985}, \"815e7116df\": {\"quality\": 0.041025641025641026, \"cost\": 0.00256722, \"time\": 102.2627355337143}, \"816068ff07\": {\"quality\": 0.15833333333333333, \"cost\": 0.0041238989999999994, \"time\": 127.76587257385255}, \"81a4f42fd9\": {\"quality\": 0.21752136752136753, \"cost\": 0.005694090000000001, \"time\": 59.030240178108215}, \"828ccea2d3\": {\"quality\": 0.04038461538461539, \"cost\": 0.008288664, \"time\": 109.34065868854523}, \"829df73946\": {\"quality\": 0.2098290598290598, \"cost\": 0.002660831, \"time\": 58.90988037586212}, \"831728b179\": {\"quality\": 0.0, \"cost\": 0.005039063999999999, \"time\": 71.040651845932}, \"831e8b8be5\": {\"quality\": 0.1987179487179487, \"cost\": 0.002362275, \"time\": 99.75399866104127}, \"8357183895\": {\"quality\": 0.0, \"cost\": 0.008517975, \"time\": 130.41329088211057}, \"8392a6083a\": {\"quality\": 0.2098290598290598, \"cost\": 0.013261482, \"time\": 129.61621696949004}, \"83b244c163\": {\"quality\": 0.2098290598290598, \"cost\": 0.0017275300000000001, \"time\": 63.67854707241058}, \"83c9e66ec6\": {\"quality\": 0.2237179487179487, \"cost\": 0.007498675, \"time\": 99.27389492988587}, \"842c0d1062\": {\"quality\": 0.20705128205128204, \"cost\": 0.005142987, \"time\": 166.0325870513916}, \"846bed2aa7\": {\"quality\": 0.24252136752136752, \"cost\": 0.006434445, \"time\": 98.26402361392975}, \"847fd49235\": {\"quality\": 0.17863247863247866, \"cost\": 0.004663961999999999, \"time\": 112.9008763551712}, \"84dc98be95\": {\"quality\": 0.19444444444444442, \"cost\": 0.007237588999999999, \"time\": 108.35145225524903}, \"8519bef585\": {\"quality\": 0.015384615384615385, \"cost\": 0.0024311339999999997, \"time\": 63.10895071029663}, \"8572c6af3a\": {\"quality\": 0.22863247863247865, \"cost\": 0.005362141000000001, \"time\": 159.52487354278566}, \"85c94a5505\": {\"quality\": 0.05, \"cost\": 0.0025202259999999995, \"time\": 34.404055738449095}, \"85e8eaed6e\": {\"quality\": 0.06538461538461539, \"cost\": 0.004890077999999999, \"time\": 158.11657013893125}, \"862183bfb9\": {\"quality\": 0.21752136752136753, \"cost\": 0.007488814999999999, \"time\": 120.0406931400299}, \"8668f65f05\": {\"quality\": 0.21752136752136753, \"cost\": 0.009651689000000001, \"time\": 144.70214662551882}, \"870e2f87b4\": {\"quality\": 0.21752136752136753, \"cost\": 0.013074957000000002, \"time\": 150.40512261390685}, \"87c1b31c82\": {\"quality\": 0.15982905982905982, \"cost\": 0.011328134, \"time\": 152.22725927829742}, \"88436e05a9\": {\"quality\": 0.09252136752136753, \"cost\": 0.010687336, \"time\": 153.95657515525818}, \"887ad124e1\": {\"quality\": 0.1987179487179487, \"cost\": 0.0030405569999999997, \"time\": 113.64234898090362}, \"8886cb3082\": {\"quality\": 0.2098290598290598, \"cost\": 0.009527662999999999, \"time\": 144.03201706409453}, \"8940398bf1\": {\"quality\": 0.04038461538461539, \"cost\": 0.001635585, \"time\": 86.64000837802887}, \"8941621423\": {\"quality\": 0.1737179487179487, \"cost\": 0.006337113, \"time\": 113.57799446582794}, \"8961e4d901\": {\"quality\": 0.0, \"cost\": 0.007177176, \"time\": 104.54208600521088}, \"8974aa89a0\": {\"quality\": 0.2098290598290598, \"cost\": 0.001461674, \"time\": 45.66577224731445}, \"89a289907e\": {\"quality\": 0.25085470085470085, \"cost\": 0.008718126999999999, \"time\": 113.69486925601959}, \"89a35a09b1\": {\"quality\": 0.21752136752136753, \"cost\": 0.007729346999999999, \"time\": 109.72029886245727}, \"89bc21961a\": {\"quality\": 0.18482905982905984, \"cost\": 0.000487962, \"time\": 54.24717800617218}, \"89fbefd150\": {\"quality\": 0.23974358974358972, \"cost\": 0.00825181, \"time\": 111.73369183540345}, \"8a37c82283\": {\"quality\": 0.23974358974358972, \"cost\": 0.007666522, \"time\": 83.14605205059051}, \"8aaadb8649\": {\"quality\": 0.025, \"cost\": 0.006027930000000001, \"time\": 66.97984294891357}, \"8acd758b7f\": {\"quality\": 0.2098290598290598, \"cost\": 0.004582077, \"time\": 86.44987049102784}, \"8b721bbc6f\": {\"quality\": 0.21752136752136753, \"cost\": 0.007828845000000001, \"time\": 117.83376302719117}, \"8b90f4b639\": {\"quality\": 0.0, \"cost\": 0.00041072999999999994, \"time\": 49.55189027786255}, \"8bbbe0f52a\": {\"quality\": 0.1814102564102564, \"cost\": 0.006133866, \"time\": 83.5189700126648}, \"8bc184f385\": {\"quality\": 0.1702991452991453, \"cost\": 0.007205481, \"time\": 86.5741204738617}, \"8bf5c3eadc\": {\"quality\": 0.025, \"cost\": 0.006170141999999999, \"time\": 83.84268100261687}, \"8c195addc7\": {\"quality\": 0.16944444444444445, \"cost\": 0.01389775, \"time\": 90.39201626777648}, \"8c9881972c\": {\"quality\": 0.0626068376068376, \"cost\": 0.0028528439999999998, \"time\": 86.04524366855621}, \"8cf8b81d84\": {\"quality\": 0.125, \"cost\": 0.0008481899999999999, \"time\": 79.2455335855484}, \"8d79e03266\": {\"quality\": 0.21752136752136753, \"cost\": 0.008664971, \"time\": 108.97852082252503}, \"8d90814b94\": {\"quality\": 0.21752136752136753, \"cost\": 0.009054151, \"time\": 130.47983191013338}, \"8e33fac90f\": {\"quality\": 0.30982905982905984, \"cost\": 0.010998730000000002, \"time\": 130.90265057086944}, \"8e5842ccbd\": {\"quality\": 0.14038461538461539, \"cost\": 0.003239019, \"time\": 70.15636944770813}, \"8f4caddfe6\": {\"quality\": 0.21752136752136753, \"cost\": 0.014952506, \"time\": 105.19670691490174}, \"8f4edde3f0\": {\"quality\": 0.3175213675213675, \"cost\": 0.017418426, \"time\": 126.13828411102295}, \"900a58f984\": {\"quality\": 0.2064102564102564, \"cost\": 0.006433365999999999, \"time\": 98.91697227954865}, \"9025e2480f\": {\"quality\": 0.1987179487179487, \"cost\": 0.006619497, \"time\": 78.82729182243347}, \"9028588af4\": {\"quality\": 0.3175213675213675, \"cost\": 0.00501719, \"time\": 37.611924695968625}, \"9059fd80ad\": {\"quality\": 0.04038461538461539, \"cost\": 0.0018876219999999998, \"time\": 75.3997132062912}, \"90d5e40c1b\": {\"quality\": 0.0, \"cost\": 0.006230459999999999, \"time\": 77.66650586128236}, \"90d9a86a2a\": {\"quality\": 0.0, \"cost\": 0.001509345, \"time\": 48.08298280239106}, \"90ed9312e1\": {\"quality\": 0.16944444444444445, \"cost\": 0.008080153999999999, \"time\": 122.39040281772614}, \"90ff13783c\": {\"quality\": 0.24316239316239316, \"cost\": 0.010701109, \"time\": 137.6493337869644}, \"90ff8eb055\": {\"quality\": 0.0702991452991453, \"cost\": 0.009148667999999999, \"time\": 137.7864047050476}, \"9104e31369\": {\"quality\": 0.06944444444444445, \"cost\": 0.012645782999999999, \"time\": 140.74590210914613}, \"918983323f\": {\"quality\": 0.12307692307692308, \"cost\": 0.0011798999999999998, \"time\": 42.641357612609866}, \"91beb0cac1\": {\"quality\": 0.125, \"cost\": 0.000758454, \"time\": 83.41916146278382}, \"91c800af6b\": {\"quality\": 0.04038461538461539, \"cost\": 0.011276811, \"time\": 143.70840940475466}, \"91dd8884db\": {\"quality\": 0.20705128205128204, \"cost\": 0.005775448000000001, \"time\": 141.27923700809478}, \"924f128b3c\": {\"quality\": 0.21752136752136753, \"cost\": 0.013985312, \"time\": 127.0183181285858}, \"9288642e53\": {\"quality\": 0.1, \"cost\": 0.006031014000000001, \"time\": 71.42156167030335}, \"92c4137fb1\": {\"quality\": 0.25085470085470085, \"cost\": 0.006908012999999999, \"time\": 145.73949379920958}, \"92c9dcd43b\": {\"quality\": 0.21752136752136753, \"cost\": 0.008693151, \"time\": 141.44435067176818}, \"92f45e5cc7\": {\"quality\": 0.23974358974358972, \"cost\": 0.0006816839999999999, \"time\": 69.65455045700074}, \"93011c0821\": {\"quality\": 0.16752136752136754, \"cost\": 0.012118556999999999, \"time\": 129.1434654712677}, \"9303149ba4\": {\"quality\": 0.04807692307692308, \"cost\": 0.007609685, \"time\": 148.0766399145126}, \"933b4d17dd\": {\"quality\": 0.22863247863247865, \"cost\": 0.004478119000000001, \"time\": 98.24304263591766}, \"94010928c6\": {\"quality\": 0.1814102564102564, \"cost\": 0.006916301999999999, \"time\": 109.81094932556152}, \"9403809e44\": {\"quality\": 0.21752136752136753, \"cost\": 0.011799824, \"time\": 132.59149780273435}, \"943baaea0c\": {\"quality\": 0.17307692307692307, \"cost\": 0.011566795000000001, \"time\": 117.94117500782014}, \"94569f177a\": {\"quality\": 0.015384615384615385, \"cost\": 0.002410395, \"time\": 125.42731764316558}, \"9466542023\": {\"quality\": 0.03333333333333333, \"cost\": 0.0057622739999999995, \"time\": 136.34905714988707}, \"947e28ef2e\": {\"quality\": 0.06538461538461539, \"cost\": 0.0031674240000000003, \"time\": 124.9436069726944}, \"94ac356663\": {\"quality\": 0.1737179487179487, \"cost\": 0.007041248, \"time\": 104.0579169511795}, \"94dff9a424\": {\"quality\": 0.1987179487179487, \"cost\": 0.005677325, \"time\": 105.50971393585205}, \"9508356a2e\": {\"quality\": 0.14807692307692308, \"cost\": 0.006912872999999999, \"time\": 129.4876657009125}, \"956bdcc254\": {\"quality\": 0.2064102564102564, \"cost\": 0.015445604, \"time\": 125.50174486637115}, \"9594b0c783\": {\"quality\": 0.2098290598290598, \"cost\": 0.003143199, \"time\": 109.28865358829498}, \"964c671f18\": {\"quality\": 0.2098290598290598, \"cost\": 0.011780327, \"time\": 147.56322202682497}, \"9679fe2b69\": {\"quality\": 0.4098290598290598, \"cost\": 0.003103125, \"time\": 112.40209789276122}, \"968fc95038\": {\"quality\": 0.21752136752136753, \"cost\": 0.00356376, \"time\": 109.65150537490845}, \"96b487c724\": {\"quality\": 0.07371794871794872, \"cost\": 0.0006113920000000001, \"time\": 69.84607322216033}, \"96e85f9af4\": {\"quality\": 0.058333333333333334, \"cost\": 0.0037689600000000005, \"time\": 115.5232797384262}, \"96f87d6483\": {\"quality\": 0.19444444444444442, \"cost\": 0.012450801, \"time\": 146.1588816165924}, \"972c83b002\": {\"quality\": 0.14807692307692308, \"cost\": 0.007372287, \"time\": 112.55022113323211}, \"975bc44958\": {\"quality\": 0.24871794871794872, \"cost\": 0.0024022649999999998, \"time\": 111.18246881961822}, \"977a4d6b6b\": {\"quality\": 0.33974358974358976, \"cost\": 0.013285368000000002, \"time\": 138.42745580673215}, \"97ad4cd41a\": {\"quality\": 0.025, \"cost\": 0.008055923000000001, \"time\": 147.88769648075103}, \"97bc30bd83\": {\"quality\": 0.19252136752136753, \"cost\": 0.010554706, \"time\": 107.51760149002075}, \"97e1d0db92\": {\"quality\": 0.06538461538461539, \"cost\": 0.004098666000000001, \"time\": 98.48549189567566}, \"981da9ba40\": {\"quality\": 0.09871794871794873, \"cost\": 0.008754687, \"time\": 150.4159719467163}, \"98c1ea89f3\": {\"quality\": 0.1987179487179487, \"cost\": 0.004756095, \"time\": 144.49644501209258}, \"98eca2c65c\": {\"quality\": 0.2098290598290598, \"cost\": 0.007576088, \"time\": 113.79488031864167}, \"98ecf1a157\": {\"quality\": 0.2098290598290598, \"cost\": 0.0006460169999999999, \"time\": 66.48974347114563}, \"9927dc270b\": {\"quality\": 0.22863247863247865, \"cost\": 0.008019299, \"time\": 142.80323901176453}, \"99546d91e4\": {\"quality\": 0.2098290598290598, \"cost\": 0.0023279909999999997, \"time\": 109.73371806144715}, \"99cb0ba736\": {\"quality\": 0.0, \"cost\": 0.001753854, \"time\": 80.39895787239075}, \"99e44ab9b2\": {\"quality\": 0.2064102564102564, \"cost\": 0.007417979999999999, \"time\": 142.21234567165374}, \"9a0145c9b5\": {\"quality\": 0.07371794871794872, \"cost\": 0.003074034, \"time\": 115.01807222366332}, \"9a57ea3f89\": {\"quality\": 0.1952991452991453, \"cost\": 0.007409801999999998, \"time\": 133.90688972473146}, \"9a8420a0b3\": {\"quality\": 0.0, \"cost\": 0.0068175779999999995, \"time\": 152.5392238378525}, \"9aa32e6c96\": {\"quality\": 0.21752136752136753, \"cost\": 0.01103525, \"time\": 141.08924725055692}, \"9aa4abfb50\": {\"quality\": 0.0, \"cost\": 0.0034209360000000003, \"time\": 89.89876456260681}, \"9ada932bf5\": {\"quality\": 0.21752136752136753, \"cost\": 0.008047750999999999, \"time\": 144.1339359998703}, \"9b6d4915f3\": {\"quality\": 0.0, \"cost\": 0.00463566, \"time\": 54.38807971477509}, \"9bae5bafc1\": {\"quality\": 0.18333333333333332, \"cost\": 0.013702148, \"time\": 98.45531895160676}, \"9c549db0a7\": {\"quality\": 0.0, \"cost\": 0.010721709000000001, \"time\": 79.0128826379776}, \"9c595a2bc9\": {\"quality\": 0.025, \"cost\": 0.007724396, \"time\": 148.95668649673462}, \"9c85f8cfcb\": {\"quality\": 0.015384615384615385, \"cost\": 0.0023850119999999997, \"time\": 67.28472025394439}, \"9c8cc46e6c\": {\"quality\": 0.06538461538461539, \"cost\": 0.002001585, \"time\": 77.71750602722167}, \"9c97d35a30\": {\"quality\": 0.2098290598290598, \"cost\": 0.005077656, \"time\": 118.16123571395875}, \"9ca354a53e\": {\"quality\": 0.0, \"cost\": 0.003021384, \"time\": 101.3728716135025}, \"9ce2c3fd98\": {\"quality\": 0.0, \"cost\": 0.0047293560000000005, \"time\": 129.50395069122314}, \"9d18cd0737\": {\"quality\": 0.07307692307692308, \"cost\": 0.004835114999999999, \"time\": 82.27081375122071}, \"9d7142e7b4\": {\"quality\": 0.16944444444444445, \"cost\": 0.008467907, \"time\": 147.60730669498443}, \"9d778daa24\": {\"quality\": 0.03333333333333333, \"cost\": 0.007418613, \"time\": 153.25245223045349}, \"9e06360bc9\": {\"quality\": 0.2098290598290598, \"cost\": 0.008218711, \"time\": 113.20028014183045}, \"9fb157be35\": {\"quality\": 0.1, \"cost\": 0.008187586, \"time\": 116.65422949790954}, \"9fc44fdeb1\": {\"quality\": 0.0, \"cost\": 0.009764671999999999, \"time\": 163.913742351532}, \"9ffaa26d5a\": {\"quality\": 0.21752136752136753, \"cost\": 0.006876598000000001, \"time\": 107.02223331928253}, \"a041e7777a\": {\"quality\": 0.17649572649572648, \"cost\": 0.0054005519999999994, \"time\": 107.31747977733612}, \"a0b81be5b4\": {\"quality\": 0.21752136752136753, \"cost\": 0.002235789, \"time\": 84.43192894458771}, \"a0c85d260e\": {\"quality\": 0.21752136752136753, \"cost\": 0.013538011000000003, \"time\": 134.37746741771696}, \"a0dc9f50ac\": {\"quality\": 0.15833333333333333, \"cost\": 0.0058347719999999985, \"time\": 73.68754653930664}, \"a14c507393\": {\"quality\": 0.0, \"cost\": 0.0016790159999999998, \"time\": 85.27930269241332}, \"a1881eb481\": {\"quality\": 0.23974358974358972, \"cost\": 0.006974355, \"time\": 158.60932910442352}, \"a1bb32e6a1\": {\"quality\": 0.04038461538461539, \"cost\": 0.0019489619999999998, \"time\": 113.17418549060821}, \"a2347e8e9e\": {\"quality\": 0.08482905982905983, \"cost\": 0.003918969, \"time\": 112.96018514633178}, \"a2aa082d14\": {\"quality\": 0.0, \"cost\": 0.012548850000000002, \"time\": 97.43302309513092}, \"a2cd339ad9\": {\"quality\": 0.09252136752136753, \"cost\": 0.009654009, \"time\": 161.78442559242248}, \"a2fd03e6a5\": {\"quality\": 0.1876068376068376, \"cost\": 0.004650046, \"time\": 46.97034850120544}, \"a31e87d7cb\": {\"quality\": 0.2064102564102564, \"cost\": 0.00146718, \"time\": 73.75970520973206}, \"a344b2d79a\": {\"quality\": 0.2098290598290598, \"cost\": 0.008522825, \"time\": 167.94721865653992}, \"a3c0ea3342\": {\"quality\": 0.3175213675213675, \"cost\": 0.007620170000000001, \"time\": 166.26401495933533}, \"a456d75fef\": {\"quality\": 0.0, \"cost\": 0.003897582, \"time\": 149.68703374862673}, \"a457f6c300\": {\"quality\": 0.0, \"cost\": 0.007946964, \"time\": 158.9001291036606}, \"a4767e7679\": {\"quality\": 0.0, \"cost\": 0.002210106, \"time\": 111.21880123615264}, \"a47de025c8\": {\"quality\": 0.0, \"cost\": 0.003860679, \"time\": 106.15438146591185}, \"a4ad96343d\": {\"quality\": 0.23696581196581196, \"cost\": 0.009205054, \"time\": 161.23487601280215}, \"a515a9c8cc\": {\"quality\": 0.16944444444444445, \"cost\": 0.0021845939999999998, \"time\": 90.79483761787415}, \"a5949b76ec\": {\"quality\": 0.0, \"cost\": 0.0022742099999999996, \"time\": 77.40827825069428}, \"a5ae4dfe66\": {\"quality\": 0.1, \"cost\": 0.006415026, \"time\": 117.52768676280975}, \"a60dd076b8\": {\"quality\": 0.1564102564102564, \"cost\": 0.0024921179999999998, \"time\": 99.6796523809433}, \"a6297a6c56\": {\"quality\": 0.125, \"cost\": 0.0019724219999999997, \"time\": 83.12536749839782}, \"a6460dbb7c\": {\"quality\": 0.2098290598290598, \"cost\": 0.002657791, \"time\": 71.54174087047576}, \"a6796ed686\": {\"quality\": 0.09444444444444444, \"cost\": 0.0032055929999999996, \"time\": 130.10447964668273}, \"a6d2b05ec8\": {\"quality\": 0.2098290598290598, \"cost\": 0.008059675, \"time\": 133.3959671497345}, \"a717c4c535\": {\"quality\": 0.11752136752136753, \"cost\": 0.012759294000000001, \"time\": 123.11250400543213}, \"a721cd9ebf\": {\"quality\": 0.21752136752136753, \"cost\": 0.006374966000000001, \"time\": 93.21334941387177}, \"a8090787b1\": {\"quality\": 0.2098290598290598, \"cost\": 0.004406745, \"time\": 124.60862724781036}, \"a854343d46\": {\"quality\": 0.225, \"cost\": 0.0063598100000000005, \"time\": 87.96014966964722}, \"a86b137d7f\": {\"quality\": 0.15, \"cost\": 0.0021869159999999997, \"time\": 58.69887585639954}, \"a88eb1493c\": {\"quality\": 0.0952991452991453, \"cost\": 0.004839059999999999, \"time\": 60.44982805252076}, \"a88fb984e3\": {\"quality\": 0.21752136752136753, \"cost\": 0.008328870999999998, \"time\": 131.90409784317018}, \"a94e2e5f57\": {\"quality\": 0.04807692307692308, \"cost\": 0.006114, \"time\": 102.40570263862611}, \"a95b4a6dd0\": {\"quality\": 0.0452991452991453, \"cost\": 0.008242788000000001, \"time\": 101.27096877098083}, \"a9621ea4e6\": {\"quality\": 0.22863247863247865, \"cost\": 0.00707055, \"time\": 100.2644911289215}, \"a96e22379d\": {\"quality\": 0.13482905982905985, \"cost\": 0.004674597, \"time\": 135.9614454984665}, \"a9721a0a50\": {\"quality\": 0.18482905982905984, \"cost\": 0.006856040000000001, \"time\": 74.35422718524933}, \"aa08180e36\": {\"quality\": 0.21752136752136753, \"cost\": 0.009903308, \"time\": 141.78917632102966}, \"aa38702a02\": {\"quality\": 0.058333333333333334, \"cost\": 0.004223736, \"time\": 108.32900211811065}, \"aa8187c023\": {\"quality\": 0.0, \"cost\": 0.006725136, \"time\": 115.54149260520936}, \"aadbfc418b\": {\"quality\": 0.04807692307692308, \"cost\": 0.003954492, \"time\": 115.65469679832458}, \"ab1c706436\": {\"quality\": 0.09871794871794873, \"cost\": 0.007816556999999998, \"time\": 157.3217898607254}, \"ab43b02cb0\": {\"quality\": 0.0, \"cost\": 0.010739148, \"time\": 88.10484538078308}, \"aba21780bc\": {\"quality\": 0.09594017094017093, \"cost\": 0.001212108, \"time\": 68.86885101795197}, \"abbca95f00\": {\"quality\": 0.1841880341880342, \"cost\": 0.00179952, \"time\": 77.29247143268586}, \"ac208e7a1d\": {\"quality\": 0.21752136752136753, \"cost\": 0.009940174000000001, \"time\": 106.78546745777129}, \"ac7fcf90e2\": {\"quality\": 0.059829059829059825, \"cost\": 0.008491644, \"time\": 163.5559736728668}, \"ac828ffe70\": {\"quality\": 0.025, \"cost\": 0.000761398, \"time\": 72.60451011657715}, \"ac9fdc1550\": {\"quality\": 0.2098290598290598, \"cost\": 0.012285289000000001, \"time\": 112.57146043777465}, \"ad328d5108\": {\"quality\": 0.1611111111111111, \"cost\": 0.001292802, \"time\": 134.6257879257202}, \"ad3efe44c3\": {\"quality\": 0.0, \"cost\": 0.002330118, \"time\": 83.25956127643585}, \"ad48432c22\": {\"quality\": 0.0, \"cost\": 0.009617565, \"time\": 165.95421760082246}, \"ad5187a390\": {\"quality\": 0.22863247863247865, \"cost\": 0.010695554, \"time\": 160.50709626674652}, \"ad6ebbba8d\": {\"quality\": 0.16752136752136754, \"cost\": 0.0075785209999999995, \"time\": 114.89612691402435}, \"ad90055ef6\": {\"quality\": 0.0, \"cost\": 0.003493998, \"time\": 103.62199125289916}, \"ad97c5cee6\": {\"quality\": 0.15149572649572648, \"cost\": 0.0022562339999999998, \"time\": 137.5054316520691}, \"adab1e0fb1\": {\"quality\": 0.3175213675213675, \"cost\": 0.00585581, \"time\": 77.00263237953186}, \"ae655ec593\": {\"quality\": 0.11666666666666667, \"cost\": 0.003635154, \"time\": 112.42764747142792}, \"ae94b172be\": {\"quality\": 0.1987179487179487, \"cost\": 0.007214128000000001, \"time\": 117.53457653522491}, \"af360c323c\": {\"quality\": 0.0, \"cost\": 0.0032059769999999996, \"time\": 116.28576538562774}, \"af90567194\": {\"quality\": 0.21752136752136753, \"cost\": 0.013109858999999998, \"time\": 144.84608309268953}, \"b0156bb6d2\": {\"quality\": 0.25085470085470085, \"cost\": 0.011064174, \"time\": 144.33307461738588}, \"b03c31ca45\": {\"quality\": 0.09871794871794873, \"cost\": 0.007032179999999999, \"time\": 159.58066523075104}, \"b0530b98c3\": {\"quality\": 0.08333333333333334, \"cost\": 0.007914135, \"time\": 151.2534171819687}, \"b0948c05b6\": {\"quality\": 0.16538461538461538, \"cost\": 0.011687544000000001, \"time\": 77.90738432407379}, \"b18168b9c1\": {\"quality\": 0.1952991452991453, \"cost\": 0.007746042, \"time\": 102.36798930168152}, \"b1cf8d33e5\": {\"quality\": 0.0702991452991453, \"cost\": 0.007221741, \"time\": 96.63562579154969}, \"b1dcd7aa24\": {\"quality\": 0.2098290598290598, \"cost\": 0.009499193, \"time\": 142.64096357822416}, \"b214718d07\": {\"quality\": 0.16538461538461538, \"cost\": 0.002830191, \"time\": 105.70124731063842}, \"b28925e4b8\": {\"quality\": 0.06538461538461539, \"cost\": 0.011244171, \"time\": 117.42571413516998}, \"b2b057ba41\": {\"quality\": 0.21752136752136753, \"cost\": 0.0036646409999999997, \"time\": 104.08598182201385}, \"b2e063499d\": {\"quality\": 0.21752136752136753, \"cost\": 0.008730451, \"time\": 99.22217960357665}, \"b3369775dc\": {\"quality\": 0.2064102564102564, \"cost\": 0.012187132, \"time\": 128.45301840305328}, \"b35bf038c2\": {\"quality\": 0.14038461538461539, \"cost\": 0.0027645659999999996, \"time\": 118.8479066848755}, \"b363b25367\": {\"quality\": 0.10982905982905983, \"cost\": 0.00807849, \"time\": 150.05790014266967}, \"b3decd5c2f\": {\"quality\": 0.14722222222222223, \"cost\": 0.0054683579999999996, \"time\": 114.51114053726197}, \"b3f20b706d\": {\"quality\": 0.025, \"cost\": 0.002810052, \"time\": 90.74897117614745}, \"b4002173ee\": {\"quality\": 0.10641025641025642, \"cost\": 0.00532104, \"time\": 71.4259075164795}, \"b45fc30d81\": {\"quality\": 0.015384615384615385, \"cost\": 0.007027874999999999, \"time\": 148.3045286655426}, \"b4a259f6dd\": {\"quality\": 0.2098290598290598, \"cost\": 0.00846508, \"time\": 139.42276346683502}, \"b52cdb3c6d\": {\"quality\": 0.25085470085470085, \"cost\": 0.001953663, \"time\": 101.56977760791779}, \"b56c312eda\": {\"quality\": 0.21752136752136753, \"cost\": 0.008759253, \"time\": 135.296657371521}, \"b5a02bb8ab\": {\"quality\": 0.09252136752136753, \"cost\": 0.005621411999999999, \"time\": 126.08018836975097}, \"b5e2b41c1c\": {\"quality\": 0.1987179487179487, \"cost\": 0.004181511, \"time\": 97.78416235446929}, \"b64ddb14f9\": {\"quality\": 0.21752136752136753, \"cost\": 0.002377122, \"time\": 40.39987435340882}, \"b66118d5f2\": {\"quality\": 0.04038461538461539, \"cost\": 0.007001360999999999, \"time\": 141.72445845603943}, \"b67107a43e\": {\"quality\": 0.0, \"cost\": 0.005282744999999999, \"time\": 134.97200748920443}, \"b682a23b89\": {\"quality\": 0.2098290598290598, \"cost\": 0.004507057, \"time\": 94.8906276702881}, \"b690a1ddd6\": {\"quality\": 0.0, \"cost\": 0.004254936000000001, \"time\": 132.1497477531433}, \"b796b7ffd3\": {\"quality\": 0.21752136752136753, \"cost\": 0.008754787, \"time\": 134.97328844070432}, \"b7d0e8557f\": {\"quality\": 0.20982905982905983, \"cost\": 0.007772019, \"time\": 136.01008360385896}, \"b7f203a0bf\": {\"quality\": 0.09871794871794873, \"cost\": 0.0074011649999999995, \"time\": 142.89838807582856}, \"b81d5b2bd9\": {\"quality\": 0.04807692307692308, \"cost\": 0.008468160999999998, \"time\": 139.87127735614774}, \"b8343f05e1\": {\"quality\": 0.04722222222222222, \"cost\": 0.0071142839999999985, \"time\": 114.42319309711456}, \"b8ab3d2f25\": {\"quality\": 0.21752136752136753, \"cost\": 0.008466568, \"time\": 101.01810977458953}, \"b8b569172f\": {\"quality\": 0.10705128205128206, \"cost\": 0.010024281999999999, \"time\": 115.06630852222443}, \"b8b91e375d\": {\"quality\": 0.21752136752136753, \"cost\": 0.012645420000000001, \"time\": 146.17070028781893}, \"b8c685904d\": {\"quality\": 0.0, \"cost\": 0.0017556779999999999, \"time\": 98.82595324516296}, \"b8d1903276\": {\"quality\": 0.025, \"cost\": 0.004277349, \"time\": 116.25438375473021}, \"b91e7fdb29\": {\"quality\": 0.0, \"cost\": 0.009623747999999998, \"time\": 172.33287427425387}, \"b9770c2261\": {\"quality\": 0.24871794871794872, \"cost\": 0.0018192599999999998, \"time\": 68.26051120758056}, \"b9bb1e6f8d\": {\"quality\": 0.21752136752136753, \"cost\": 0.012109797999999998, \"time\": 153.9570437669754}, \"b9da208432\": {\"quality\": 0.22094017094017093, \"cost\": 0.009226414999999998, \"time\": 152.71056313514708}, \"ba3223f6ac\": {\"quality\": 0.17307692307692307, \"cost\": 0.003906612, \"time\": 117.72781562805176}, \"bac3d23c31\": {\"quality\": 0.21752136752136753, \"cost\": 0.011714114000000001, \"time\": 149.72536618709563}, \"bb3ee18de1\": {\"quality\": 0.2064102564102564, \"cost\": 0.00687103, \"time\": 112.67253947257996}, \"bb6536b0ab\": {\"quality\": 0.12585470085470085, \"cost\": 0.010593946, \"time\": 116.58704767227172}, \"bb70f60bf1\": {\"quality\": 0.1952991452991453, \"cost\": 0.0029260320000000003, \"time\": 113.47788968086243}, \"bbba9dd6ae\": {\"quality\": 0.3175213675213675, \"cost\": 0.005866453000000001, \"time\": 81.5745332479477}, \"bbfba2f2ee\": {\"quality\": 0.0, \"cost\": 0.0043119479999999995, \"time\": 144.39048359394076}, \"bc3d02f753\": {\"quality\": 0.0, \"cost\": 0.002622999, \"time\": 100.0285652399063}, \"bc4c1fcc64\": {\"quality\": 0.21752136752136753, \"cost\": 0.005892321000000001, \"time\": 80.69622204303741}, \"bc60556255\": {\"quality\": 0.025, \"cost\": 0.000541426, \"time\": 90.43072290420533}, \"bcae7c2fc4\": {\"quality\": 0.0952991452991453, \"cost\": 0.006106959, \"time\": 118.90915808677673}, \"bcef42e3b0\": {\"quality\": 0.20705128205128204, \"cost\": 0.003036438, \"time\": 126.3029891014099}, \"bcf4bf7c35\": {\"quality\": 0.09252136752136753, \"cost\": 0.010682502, \"time\": 163.63021724224092}, \"bcfb273436\": {\"quality\": 0.2675213675213675, \"cost\": 0.005338022999999999, \"time\": 154.81484963893888}, \"bd99b2fb21\": {\"quality\": 0.21752136752136753, \"cost\": 0.00784732, \"time\": 116.22147076129913}, \"bdf497196b\": {\"quality\": 0.21752136752136753, \"cost\": 0.00564954, \"time\": 81.09978458881378}, \"be2ae88f70\": {\"quality\": 0.0, \"cost\": 0.004892166, \"time\": 134.319625210762}, \"be4740f38f\": {\"quality\": 0.11752136752136753, \"cost\": 0.011724527999999998, \"time\": 111.63968887329102}, \"bed888d4dc\": {\"quality\": 0.12222222222222223, \"cost\": 0.007974548999999997, \"time\": 167.2181126832962}, \"bf2a5d2680\": {\"quality\": 0.08141025641025641, \"cost\": 0.005895564, \"time\": 116.7006804227829}, \"bf7b0a8dc1\": {\"quality\": 0.21752136752136753, \"cost\": 0.0072987239999999995, \"time\": 172.42680845260622}, \"bfed7670ed\": {\"quality\": 0.08482905982905983, \"cost\": 0.010263118, \"time\": 118.4290988445282}, \"c06b118e65\": {\"quality\": 0.20705128205128204, \"cost\": 0.005422905, \"time\": 111.81110198497773}, \"c08a5ad170\": {\"quality\": 0.19252136752136753, \"cost\": 0.008962954000000002, \"time\": 153.46751430034638}, \"c0d53a20de\": {\"quality\": 0.2098290598290598, \"cost\": 0.007020855, \"time\": 151.7084014415741}, \"c10e588987\": {\"quality\": 0.1814102564102564, \"cost\": 0.0070173779999999995, \"time\": 161.47480256557463}, \"c127509a7a\": {\"quality\": 0.30982905982905984, \"cost\": 0.012152782, \"time\": 114.87017834186554}, \"c13682c7c7\": {\"quality\": 0.21752136752136753, \"cost\": 0.011820473999999997, \"time\": 155.98555040359497}, \"c13d6e78e9\": {\"quality\": 0.2098290598290598, \"cost\": 0.015482613, \"time\": 135.19117062091829}, \"c145482664\": {\"quality\": 0.125, \"cost\": 0.011503632, \"time\": 157.4289677143097}, \"c14ff3144d\": {\"quality\": 0.15833333333333333, \"cost\": 0.001474128, \"time\": 73.17416946887971}, \"c186182658\": {\"quality\": 0.0702991452991453, \"cost\": 0.00651766, \"time\": 114.43175661563873}, \"c263c65d0a\": {\"quality\": 0.2098290598290598, \"cost\": 0.004897752, \"time\": 161.65783095359802}, \"c2949aa902\": {\"quality\": 0.2064102564102564, \"cost\": 0.006985964999999999, \"time\": 112.66284742355347}, \"c31c9d4d8c\": {\"quality\": 0.058333333333333334, \"cost\": 0.004036233, \"time\": 118.73210837841035}, \"c31e956b35\": {\"quality\": 0.10705128205128206, \"cost\": 0.008987883, \"time\": 156.68281931877135}, \"c339463a25\": {\"quality\": 0.11538461538461539, \"cost\": 0.0039407339999999996, \"time\": 142.9191876411438}, \"c36b525dde\": {\"quality\": 0.2098290598290598, \"cost\": 0.00295329, \"time\": 84.8311204433441}, \"c3d20f33bf\": {\"quality\": 0.15085470085470087, \"cost\": 0.009346874, \"time\": 155.06189365386962}, \"c3ec2cec59\": {\"quality\": 0.3175213675213675, \"cost\": 0.010893408, \"time\": 121.71417863368988}, \"c443a2c1fb\": {\"quality\": 0.09444444444444444, \"cost\": 0.008210058, \"time\": 158.09974863529203}, \"c4a80d19b3\": {\"quality\": 0.3175213675213675, \"cost\": 0.010017962999999998, \"time\": 152.05399761199953}, \"c4c2826afd\": {\"quality\": 0.14722222222222223, \"cost\": 0.008821047, \"time\": 150.96840312480924}, \"c4c94a5527\": {\"quality\": 0.14807692307692308, \"cost\": 0.007312788, \"time\": 118.44996173381804}, \"c4f3e7665d\": {\"quality\": 0.1, \"cost\": 0.004909806, \"time\": 122.77528305053711}, \"c58e9652b7\": {\"quality\": 0.03333333333333333, \"cost\": 0.004275492, \"time\": 163.37369763851166}, \"c5a16b834a\": {\"quality\": 0.24252136752136752, \"cost\": 0.009288249, \"time\": 145.43366010189055}, \"c5fbe2076f\": {\"quality\": 0.21752136752136753, \"cost\": 0.016178868, \"time\": 153.44052112102509}, \"c6198f364e\": {\"quality\": 0.21752136752136753, \"cost\": 0.008840571000000002, \"time\": 143.01735095977784}, \"c691570715\": {\"quality\": 0.025, \"cost\": 0.004222068, \"time\": 105.88032670021057}, \"c691a29c42\": {\"quality\": 0.06944444444444445, \"cost\": 0.014146588, \"time\": 136.24555165767669}, \"c6a339987c\": {\"quality\": 0.14807692307692308, \"cost\": 0.008871827000000002, \"time\": 124.18036108016967}, \"c6a4d256ce\": {\"quality\": 0.11752136752136753, \"cost\": 0.011863845, \"time\": 131.42284343242645}, \"c76222087e\": {\"quality\": 0.10982905982905983, \"cost\": 0.007390348000000001, \"time\": 108.10401859283448}, \"c772ff3704\": {\"quality\": 0.10982905982905983, \"cost\": 0.008256624, \"time\": 122.45812864303589}, \"c7d4ff0c05\": {\"quality\": 0.22863247863247865, \"cost\": 0.006689247, \"time\": 135.3987812280655}, \"c823f7ab29\": {\"quality\": 0.19252136752136753, \"cost\": 0.006112934, \"time\": 110.04468183517456}, \"c82b926689\": {\"quality\": 0.21752136752136753, \"cost\": 0.011001155000000002, \"time\": 132.11898975372316}, \"c82f834e85\": {\"quality\": 0.21752136752136753, \"cost\": 0.007510131000000001, \"time\": 88.01416339874268}, \"c9320068f9\": {\"quality\": 0.11474358974358972, \"cost\": 0.008706341999999999, \"time\": 138.78494324684144}, \"c935a33384\": {\"quality\": 0.23974358974358972, \"cost\": 0.013361947999999998, \"time\": 128.84306230545045}, \"c99f3577c7\": {\"quality\": 0.09444444444444444, \"cost\": 0.012227506, \"time\": 131.7928378343582}, \"c9d32a0a82\": {\"quality\": 0.1, \"cost\": 0.00772979, \"time\": 150.54008300304412}, \"ca55c36c3f\": {\"quality\": 0.2098290598290598, \"cost\": 0.002033097, \"time\": 109.44059422016144}, \"caa7c0bd6b\": {\"quality\": 0.1737179487179487, \"cost\": 0.007166354999999999, \"time\": 108.27636570930481}, \"cac6b051e9\": {\"quality\": 0.2098290598290598, \"cost\": 0.007765482000000001, \"time\": 140.63682539463042}, \"cb19d631b2\": {\"quality\": 0.2098290598290598, \"cost\": 0.0074525500000000005, \"time\": 109.26569464206696}, \"cbb25fb322\": {\"quality\": 0.19444444444444442, \"cost\": 0.004088256, \"time\": 146.00942890644075}, \"cbb5eb0e74\": {\"quality\": 0.2098290598290598, \"cost\": 0.007899888, \"time\": 146.89739501476288}, \"cbc32cbeff\": {\"quality\": 0.2098290598290598, \"cost\": 0.008663157000000001, \"time\": 144.39441390037535}, \"cbd4461293\": {\"quality\": 0.09871794871794873, \"cost\": 0.0019465299999999997, \"time\": 53.78504421710968}, \"cbe2318045\": {\"quality\": 0.18482905982905984, \"cost\": 0.00824068, \"time\": 99.53602702617644}, \"cc20ebc768\": {\"quality\": 0.13974358974358975, \"cost\": 0.008763393999999999, \"time\": 151.47476081848146}, \"cc886fe337\": {\"quality\": 0.05, \"cost\": 0.006118446, \"time\": 104.02940430641175}, \"ccf72745c1\": {\"quality\": 0.21752136752136753, \"cost\": 0.00881063, \"time\": 104.33854112625122}, \"cd1d418732\": {\"quality\": 0.0, \"cost\": 0.005609648, \"time\": 51.80708644390106}, \"cd23c79db1\": {\"quality\": 0.0, \"cost\": 0.008759461999999999, \"time\": 130.19892246723174}, \"cd64fbfcd9\": {\"quality\": 0.02222222222222222, \"cost\": 0.006661377, \"time\": 132.06611769199372}, \"cda3e2a4e9\": {\"quality\": 0.19252136752136753, \"cost\": 0.003251412, \"time\": 130.11529834270476}, \"cdc9ce922f\": {\"quality\": 0.1876068376068376, \"cost\": 0.00361428, \"time\": 130.60064585208892}, \"cdf0df2f51\": {\"quality\": 0.16538461538461538, \"cost\": 0.005876955, \"time\": 170.43609290122987}, \"ce281875b4\": {\"quality\": 0.2098290598290598, \"cost\": 0.002271957, \"time\": 134.98369066715242}, \"ce4bc5f348\": {\"quality\": 0.04038461538461539, \"cost\": 0.0031947119999999997, \"time\": 97.7704703092575}, \"ce980cf86f\": {\"quality\": 0.125, \"cost\": 0.003918672, \"time\": 130.57520353794098}, \"cecca90dd2\": {\"quality\": 0.1987179487179487, \"cost\": 0.004469783999999999, \"time\": 136.02725315093994}, \"cece83de2d\": {\"quality\": 0.10982905982905983, \"cost\": 0.0060226260000000005, \"time\": 134.92675962448118}, \"cf51e0a888\": {\"quality\": 0.025, \"cost\": 0.007815034000000002, \"time\": 141.21634793281555}, \"cf9538faf0\": {\"quality\": 0.21752136752136753, \"cost\": 0.009526646, \"time\": 136.5719269990921}, \"cf9d2e224c\": {\"quality\": 0.11752136752136753, \"cost\": 0.005909796, \"time\": 82.14293384552002}, \"cfd36f3a8c\": {\"quality\": 0.2064102564102564, \"cost\": 0.015834602, \"time\": 133.97862401008604}, \"cffa29a6ef\": {\"quality\": 0.1987179487179487, \"cost\": 0.00053197, \"time\": 57.04082276821137}, \"d03596c3de\": {\"quality\": 0.23974358974358972, \"cost\": 0.007832440999999999, \"time\": 161.64306690692902}, \"d0f9633442\": {\"quality\": 0.0, \"cost\": 0.007817082, \"time\": 133.32561285495757}, \"d192872a51\": {\"quality\": 0.06944444444444445, \"cost\": 0.011960976000000002, \"time\": 151.55924673080443}, \"d1d953cac7\": {\"quality\": 0.023076923076923078, \"cost\": 0.0069616439999999995, \"time\": 129.82648131847384}, \"d2164c8c4c\": {\"quality\": 0.25085470085470085, \"cost\": 0.007942218000000001, \"time\": 106.56984751224518}, \"d216eab7d8\": {\"quality\": 0.23974358974358972, \"cost\": 0.008338906, \"time\": 110.89031755924225}, \"d266c19ac8\": {\"quality\": 0.18482905982905984, \"cost\": 0.005187306, \"time\": 137.3413758277893}, \"d2af24b59e\": {\"quality\": 0.19252136752136753, \"cost\": 0.008368941999999999, \"time\": 107.03122828006744}, \"d3a2d50bd7\": {\"quality\": 0.2098290598290598, \"cost\": 0.004220073, \"time\": 104.34660010337831}, \"d3db4cf84d\": {\"quality\": 0.0, \"cost\": 0.0043255260000000005, \"time\": 86.13068716526033}, \"d40174fb0b\": {\"quality\": 0.015384615384615385, \"cost\": 0.004637364, \"time\": 72.65830745697022}, \"d43fafa19e\": {\"quality\": 0.023076923076923078, \"cost\": 0.004984457999999999, \"time\": 69.79157807826996}, \"d48ead13da\": {\"quality\": 0.04807692307692308, \"cost\": 0.007356842000000001, \"time\": 108.29052205085753}, \"d5016f4538\": {\"quality\": 0.023076923076923078, \"cost\": 0.004924038, \"time\": 63.6337170124054}, \"d55e983189\": {\"quality\": 0.2098290598290598, \"cost\": 0.0023029619999999995, \"time\": 103.95245385169983}, \"d573c2a414\": {\"quality\": 0.04444444444444444, \"cost\": 0.003798615, \"time\": 146.4341695547104}, \"d58036ba66\": {\"quality\": 0.21752136752136753, \"cost\": 0.007977542, \"time\": 101.040083694458}, \"d59bacbfe0\": {\"quality\": 0.09252136752136753, \"cost\": 0.009014214999999999, \"time\": 152.89927747249604}, \"d6040140b9\": {\"quality\": 0.21752136752136753, \"cost\": 0.013529567000000001, \"time\": 147.0174176454544}, \"d640edd7a7\": {\"quality\": 0.2064102564102564, \"cost\": 0.0030975000000000004, \"time\": 118.49127612113952}, \"d65185c1a4\": {\"quality\": 0.2098290598290598, \"cost\": 0.0048733019999999995, \"time\": 101.63841800689698}, \"d690b6d739\": {\"quality\": 0.13760683760683762, \"cost\": 0.0024723749999999997, \"time\": 115.86975026130676}, \"d6bd3b66ba\": {\"quality\": 0.0, \"cost\": 0.0037659959999999998, \"time\": 111.267901968956}, \"d6c4e48eeb\": {\"quality\": 0.21752136752136753, \"cost\": 0.008788099, \"time\": 116.89563345909119}, \"d6c60a5214\": {\"quality\": 0.17307692307692307, \"cost\": 0.0012564660000000001, \"time\": 69.47750086784363}, \"d6cbf265ee\": {\"quality\": 0.06538461538461539, \"cost\": 0.00160378, \"time\": 72.91289446353912}, \"d73a9aab4e\": {\"quality\": 0.04871794871794872, \"cost\": 0.0032188139999999995, \"time\": 79.5758284330368}, \"d752c30d07\": {\"quality\": 0.3175213675213675, \"cost\": 0.015617612, \"time\": 127.57117984294892}, \"d7c0972014\": {\"quality\": 0.19444444444444442, \"cost\": 0.0105163, \"time\": 80.39488637447357}, \"d7f6c0c9d4\": {\"quality\": 0.0, \"cost\": 0.002123865, \"time\": 94.49595766067506}, \"d813410e44\": {\"quality\": 0.04807692307692308, \"cost\": 0.002987295, \"time\": 133.61272318363189}, \"d87eb775da\": {\"quality\": 0.2098290598290598, \"cost\": 0.008013359000000001, \"time\": 119.17614867687226}, \"d8bab6c09b\": {\"quality\": 0.07307692307692308, \"cost\": 0.011752513, \"time\": 155.0189305305481}, \"d8bcac36e8\": {\"quality\": 0.025, \"cost\": 0.010723482, \"time\": 115.70462930202484}, \"d96677d8d4\": {\"quality\": 0.21752136752136753, \"cost\": 0.006440180999999999, \"time\": 108.42673063278198}, \"d9e2bb21a3\": {\"quality\": 0.19594017094017094, \"cost\": 0.005149365, \"time\": 107.66626167297363}, \"daaadadcc9\": {\"quality\": 0.21752136752136753, \"cost\": 0.009858946, \"time\": 142.47570564746857}, \"daf855e065\": {\"quality\": 0.1987179487179487, \"cost\": 0.005835996000000001, \"time\": 153.1451871395111}, \"db6f7259cd\": {\"quality\": 0.17307692307692307, \"cost\": 0.007230853000000001, \"time\": 107.38680810928344}, \"db9060cd27\": {\"quality\": 0.025, \"cost\": 0.00183247, \"time\": 75.05918591022493}, \"dbce95a072\": {\"quality\": 0.10641025641025642, \"cost\": 0.0072439719999999996, \"time\": 107.68909630775451}, \"dbe7d818fa\": {\"quality\": 0.14444444444444443, \"cost\": 0.0020769660000000004, \"time\": 109.260413813591}, \"dc66bccb1c\": {\"quality\": 0.2098290598290598, \"cost\": 0.00518188, \"time\": 101.09860997200012}, \"dc90065dea\": {\"quality\": 0.11752136752136753, \"cost\": 0.010587499, \"time\": 108.61172733306884}, \"dc9a912501\": {\"quality\": 0.06538461538461539, \"cost\": 0.004132665, \"time\": 122.22055804729462}, \"dd0d70fedd\": {\"quality\": 0.1987179487179487, \"cost\": 0.0018010649999999997, \"time\": 72.4727225780487}, \"dd76899626\": {\"quality\": 0.0, \"cost\": 0.01104418, \"time\": 163.46480329036712}, \"dd9f5d1ba9\": {\"quality\": 0.1987179487179487, \"cost\": 0.003202473, \"time\": 168.22212414741517}, \"de1645053b\": {\"quality\": 0.21752136752136753, \"cost\": 0.009485828, \"time\": 156.01319019794465}, \"de18bf45e1\": {\"quality\": 0.03333333333333333, \"cost\": 0.001299202, \"time\": 79.47014775276185}, \"de1e56370f\": {\"quality\": 0.2098290598290598, \"cost\": 0.0039003330000000006, \"time\": 95.9440367937088}, \"deb84ddd06\": {\"quality\": 0.0, \"cost\": 0.006051836, \"time\": 89.6067531824112}, \"df2bb408cf\": {\"quality\": 0.125, \"cost\": 0.0037215599999999996, \"time\": 133.46323487758636}, \"df2ebe2c01\": {\"quality\": 0.1987179487179487, \"cost\": 0.0045562350000000005, \"time\": 174.54163272380828}, \"dfb8aebe38\": {\"quality\": 0.21752136752136753, \"cost\": 0.008408229, \"time\": 166.86121113300322}, \"dfce6153aa\": {\"quality\": 0.08482905982905983, \"cost\": 0.00791217, \"time\": 165.6665771961212}, \"dfda94bd2a\": {\"quality\": 0.0, \"cost\": 0.0017028, \"time\": 85.28198800086975}, \"dff452a9ca\": {\"quality\": 0.0, \"cost\": 0.005131782, \"time\": 116.72450284957885}, \"e02f982a26\": {\"quality\": 0.2064102564102564, \"cost\": 0.00861959, \"time\": 133.95475313663482}, \"e06701b665\": {\"quality\": 0.07222222222222222, \"cost\": 0.005247648, \"time\": 155.51543612480162}, \"e18abd2ab0\": {\"quality\": 0.19252136752136753, \"cost\": 0.008510738, \"time\": 142.51286814212799}, \"e1e596ee1b\": {\"quality\": 0.06944444444444445, \"cost\": 0.009328152, \"time\": 170.3236501932144}, \"e20ba014a1\": {\"quality\": 0.25085470085470085, \"cost\": 0.010510256999999999, \"time\": 149.4374349117279}, \"e223700849\": {\"quality\": 0.10982905982905983, \"cost\": 0.002881368, \"time\": 120.50577642917634}, \"e26c7bfbdb\": {\"quality\": 0.08141025641025641, \"cost\": 0.007298063999999999, \"time\": 115.83573853969574}, \"e35e5f81a7\": {\"quality\": 0.0, \"cost\": 0.006075344999999999, \"time\": 117.27980465888976}, \"e3cdc0d870\": {\"quality\": 0.09444444444444444, \"cost\": 0.00339135, \"time\": 131.6535663843155}, \"e3df4cf041\": {\"quality\": 0.06538461538461539, \"cost\": 0.007586271, \"time\": 106.25668971538545}, \"e3eca9854c\": {\"quality\": 0.10705128205128206, \"cost\": 0.004585331999999999, \"time\": 159.58829066753387}, \"e47dc3abca\": {\"quality\": 0.09252136752136753, \"cost\": 0.011292871999999999, \"time\": 107.18091866970062}, \"e495ff601f\": {\"quality\": 0.14807692307692308, \"cost\": 0.007825082, \"time\": 156.01348607540132}, \"e4b9d4fb41\": {\"quality\": 0.14038461538461539, \"cost\": 0.007481999, \"time\": 112.45064301490783}, \"e510bda989\": {\"quality\": 0.2098290598290598, \"cost\": 0.005625576, \"time\": 35.45525462627411}, \"e515bc1935\": {\"quality\": 0.2098290598290598, \"cost\": 0.00642271, \"time\": 124.40159273147583}, \"e517cd2222\": {\"quality\": 0.1737179487179487, \"cost\": 0.0015429100000000002, \"time\": 54.5986225605011}, \"e51b01f418\": {\"quality\": 0.1952991452991453, \"cost\": 0.005621012999999999, \"time\": 119.1760358095169}, \"e520dfae5b\": {\"quality\": 0.2098290598290598, \"cost\": 0.005809304999999999, \"time\": 154.96505286693574}, \"e53906f84b\": {\"quality\": 0.17649572649572648, \"cost\": 0.003217785, \"time\": 119.95617558956147}, \"e53b349cce\": {\"quality\": 0.04807692307692308, \"cost\": 0.006779802, \"time\": 116.54631357192993}, \"e5401ed278\": {\"quality\": 0.1987179487179487, \"cost\": 0.009414028, \"time\": 154.65711226463316}, \"e56a16ca66\": {\"quality\": 0.3237179487179487, \"cost\": 0.001822221, \"time\": 82.61048684120178}, \"e5a2f72b30\": {\"quality\": 0.2098290598290598, \"cost\": 0.008565615, \"time\": 130.87860839366914}, \"e5a70a13ac\": {\"quality\": 0.19252136752136753, \"cost\": 0.007232057000000002, \"time\": 115.96633384227752}, \"e5fdeb4de9\": {\"quality\": 0.1987179487179487, \"cost\": 0.0035100910000000003, \"time\": 106.90362923145295}, \"e609601eee\": {\"quality\": 0.13333333333333333, \"cost\": 0.001958406, \"time\": 89.7593854188919}, \"e6f141cc8f\": {\"quality\": 0.1737179487179487, \"cost\": 0.007898352, \"time\": 159.6583716392517}, \"e7525c117a\": {\"quality\": 0.125, \"cost\": 0.003349254, \"time\": 129.90280907154084}, \"e7e94ab7a5\": {\"quality\": 0.1, \"cost\": 0.001623004, \"time\": 95.54465639591217}, \"e86bd256d6\": {\"quality\": 0.12863247863247865, \"cost\": 0.003572466, \"time\": 121.87137434482574}, \"e89283a4d9\": {\"quality\": 0.17222222222222222, \"cost\": 0.0017132999999999998, \"time\": 125.78474328517913}, \"e9befb80e0\": {\"quality\": 0.10982905982905983, \"cost\": 0.0064075799999999995, \"time\": 125.61113278865814}, \"ea38031fc1\": {\"quality\": 0.08482905982905983, \"cost\": 0.007648266000000001, \"time\": 158.89819700717925}, \"ea6ecc5653\": {\"quality\": 0.0626068376068376, \"cost\": 0.007534413, \"time\": 126.29475154876708}, \"ea8bcb3ae2\": {\"quality\": 0.16944444444444445, \"cost\": 0.01714652, \"time\": 120.57036209106445}, \"ea91a2e78b\": {\"quality\": 0.15, \"cost\": 0.005214324, \"time\": 129.3872970342636}, \"eac300f0d1\": {\"quality\": 0.1737179487179487, \"cost\": 0.008836863, \"time\": 174.77539343833922}, \"ebaa9b1297\": {\"quality\": 0.2098290598290598, \"cost\": 0.001620009, \"time\": 84.55729002952575}, \"ebbe8b6c4f\": {\"quality\": 0.0, \"cost\": 0.006925624, \"time\": 89.16193358898164}, \"ebdf3abff2\": {\"quality\": 0.10982905982905983, \"cost\": 0.013354845, \"time\": 123.62594261169434}, \"ebee1f2761\": {\"quality\": 0.22863247863247865, \"cost\": 0.004503177, \"time\": 143.60831434726714}, \"ec8844a5ae\": {\"quality\": 0.025, \"cost\": 0.00061108, \"time\": 93.01526029109955}, \"ecb5f78f37\": {\"quality\": 0.0, \"cost\": 0.010665879, \"time\": 184.15539872646332}, \"ece7ff5129\": {\"quality\": 0.09871794871794873, \"cost\": 0.006345035999999999, \"time\": 107.36751945018767}, \"ed60b8cac5\": {\"quality\": 0.025, \"cost\": 0.005388696, \"time\": 182.1943165063858}, \"ed6b5480a5\": {\"quality\": 0.0702991452991453, \"cost\": 0.004838202, \"time\": 76.91644642353057}, \"eda630dc85\": {\"quality\": 0.21752136752136753, \"cost\": 0.0075326360000000005, \"time\": 115.44470648765564}, \"edaaee5ed4\": {\"quality\": 0.07307692307692308, \"cost\": 0.006511264000000001, \"time\": 83.00507278442383}, \"edb2b764aa\": {\"quality\": 0.08482905982905983, \"cost\": 0.013305772, \"time\": 130.48116376399994}, \"edc52339db\": {\"quality\": 0.04444444444444444, \"cost\": 0.012820474000000002, \"time\": 177.77201774120329}, \"ee46042c5d\": {\"quality\": 0.14444444444444443, \"cost\": 0.011933044, \"time\": 173.3385812520981}, \"ee855899d8\": {\"quality\": 0.19444444444444442, \"cost\": 0.002078877, \"time\": 131.02240426540374}, \"eef12d478b\": {\"quality\": 0.06752136752136753, \"cost\": 0.005545968, \"time\": 137.4901770591736}, \"ef37b3e0be\": {\"quality\": 0.0, \"cost\": 0.008727486, \"time\": 182.45494146347045}, \"ef43d497f1\": {\"quality\": 0.13974358974358975, \"cost\": 0.012247042, \"time\": 182.13725912570953}, \"ef4d4c4a62\": {\"quality\": 0.3175213675213675, \"cost\": 0.008570482, \"time\": 138.68434212207794}, \"f0655621af\": {\"quality\": 0.05641025641025641, \"cost\": 0.004645079999999999, \"time\": 51.96998543739319}, \"f076b4c9ae\": {\"quality\": 0.1987179487179487, \"cost\": 0.008021229, \"time\": 145.75664427280424}, \"f07881a734\": {\"quality\": 0.0, \"cost\": 0.0008367299999999999, \"time\": 121.20072700977326}, \"f0829510fc\": {\"quality\": 0.12585470085470085, \"cost\": 0.007723235999999999, \"time\": 174.65302150249482}, \"f11eddb4ed\": {\"quality\": 0.22863247863247865, \"cost\": 0.011729255, \"time\": 164.10010514259338}, \"f12622d3d7\": {\"quality\": 0.0, \"cost\": 0.008399868000000001, \"time\": 176.69494199752808}, \"f1408da253\": {\"quality\": 0.2098290598290598, \"cost\": 0.009230984999999999, \"time\": 164.57058210372924}, \"f1770e7d28\": {\"quality\": 0.2098290598290598, \"cost\": 0.003513861, \"time\": 159.09384961128234}, \"f18cf41929\": {\"quality\": 0.0, \"cost\": 0.003205404, \"time\": 109.51144468784332}, \"f1bda127f6\": {\"quality\": 0.21752136752136753, \"cost\": 0.012127632, \"time\": 119.53270018100739}, \"f2a2e91541\": {\"quality\": 0.1814102564102564, \"cost\": 0.008741034, \"time\": 164.1866457939148}, \"f2c04ed1c8\": {\"quality\": 0.04038461538461539, \"cost\": 0.004816446, \"time\": 76.6830270767212}, \"f2cf5db12d\": {\"quality\": 0.09252136752136753, \"cost\": 0.008174008, \"time\": 126.23963840007782}, \"f3c7a062f7\": {\"quality\": 0.18482905982905984, \"cost\": 0.003666486, \"time\": 122.12519326210023}, \"f42277df7f\": {\"quality\": 0.2098290598290598, \"cost\": 0.001627851, \"time\": 72.52343001365662}, \"f437481e3b\": {\"quality\": 0.21752136752136753, \"cost\": 0.011085886000000001, \"time\": 157.97356908321382}, \"f487340019\": {\"quality\": 0.21816239316239316, \"cost\": 0.0031079669999999997, \"time\": 131.89823365211487}, \"f497b83523\": {\"quality\": 0.015384615384615385, \"cost\": 0.007367326, \"time\": 130.5487345457077}, \"f4aa8ffdf5\": {\"quality\": 0.18333333333333332, \"cost\": 0.00532851, \"time\": 143.96070950031282}, \"f4ab4a73b6\": {\"quality\": 0.07307692307692308, \"cost\": 0.0073043940000000005, \"time\": 143.23261981010438}, \"f4beb148d0\": {\"quality\": 0.2098290598290598, \"cost\": 0.0023591489999999996, \"time\": 131.58125035762788}, \"f4deb72db6\": {\"quality\": 0.0, \"cost\": 0.007903169999999998, \"time\": 146.17728536129}, \"f50a47a0aa\": {\"quality\": 0.15833333333333333, \"cost\": 0.003946065, \"time\": 136.4745246887207}, \"f5b9a94dcc\": {\"quality\": 0.0, \"cost\": 0.002980708, \"time\": 114.85360951423645}, \"f5c27e7172\": {\"quality\": 0.015384615384615385, \"cost\": 0.003205683, \"time\": 136.12301714420317}, \"f5e53d963b\": {\"quality\": 0.07222222222222222, \"cost\": 0.001996362, \"time\": 87.49879748821257}, \"f5ecac6743\": {\"quality\": 0.23974358974358972, \"cost\": 0.008653056000000001, \"time\": 190.25079679489136}, \"f614235c15\": {\"quality\": 0.1, \"cost\": 0.001451158, \"time\": 103.29747097492219}, \"f627bff3a1\": {\"quality\": 0.08333333333333334, \"cost\": 0.002846523, \"time\": 154.48251981735228}, \"f70e54a9a0\": {\"quality\": 0.2098290598290598, \"cost\": 0.008556343000000001, \"time\": 184.88659045696258}, \"f74c5a862e\": {\"quality\": 0.0, \"cost\": 0.006900402, \"time\": 186.4667979478836}, \"f74ec023e4\": {\"quality\": 0.19252136752136753, \"cost\": 0.011753, \"time\": 179.87489347457887}, \"f783b2b34a\": {\"quality\": 0.1987179487179487, \"cost\": 0.00525324, \"time\": 161.1631241083145}, \"f7b048bd54\": {\"quality\": 0.19444444444444442, \"cost\": 0.008355386999999999, \"time\": 145.28592591285707}, \"f7c4df993e\": {\"quality\": 0.2098290598290598, \"cost\": 0.007574031, \"time\": 157.67136673927308}, \"f848813dca\": {\"quality\": 0.02222222222222222, \"cost\": 0.0046792710000000005, \"time\": 184.49361929893493}, \"f854533145\": {\"quality\": 0.21752136752136753, \"cost\": 0.010477562999999999, \"time\": 188.20525524616244}, \"f8f946b5fb\": {\"quality\": 0.032692307692307694, \"cost\": 0.007302053999999999, \"time\": 199.7285678625107}, \"f912592c8d\": {\"quality\": 0.21752136752136753, \"cost\": 0.009682198999999999, \"time\": 187.25646502971648}, \"f93d9a2693\": {\"quality\": 0.10705128205128206, \"cost\": 0.006198954, \"time\": 124.54699866771698}, \"f99096d89c\": {\"quality\": 0.21752136752136753, \"cost\": 0.008768471, \"time\": 151.46780948638917}, \"f9e8e221f3\": {\"quality\": 0.06944444444444445, \"cost\": 0.006739169999999999, \"time\": 118.0668850183487}, \"fa38879eab\": {\"quality\": 0.17307692307692307, \"cost\": 0.013126675, \"time\": 162.19450600147246}, \"fa5b473f15\": {\"quality\": 0.0, \"cost\": 0.007688484000000001, \"time\": 179.80013384819028}, \"fa7882d46b\": {\"quality\": 0.21752136752136753, \"cost\": 0.011772778000000001, \"time\": 164.46603999137878}, \"fa906520d1\": {\"quality\": 0.2098290598290598, \"cost\": 0.012046285, \"time\": 162.0444113969803}, \"faabebaa30\": {\"quality\": 0.21752136752136753, \"cost\": 0.009497244000000002, \"time\": 166.34633650779722}, \"fb0339a7d0\": {\"quality\": 0.1952991452991453, \"cost\": 0.012292273000000001, \"time\": 170.6160633802414}, \"fb0e796cd3\": {\"quality\": 0.15149572649572648, \"cost\": 0.005093987999999999, \"time\": 133.95573093891142}, \"fb216ad6b3\": {\"quality\": 0.04038461538461539, \"cost\": 0.0032487699999999998, \"time\": 63.50213906764984}, \"fb29372712\": {\"quality\": 0.21752136752136753, \"cost\": 0.007952629, \"time\": 162.53553793430328}, \"fb6216880a\": {\"quality\": 0.0, \"cost\": 0.010096947, \"time\": 167.64952411651612}, \"fbd6c45271\": {\"quality\": 0.21752136752136753, \"cost\": 0.009907777, \"time\": 157.9287733793259}, \"fc1fd5bf54\": {\"quality\": 0.32863247863247863, \"cost\": 0.0051455070000000006, \"time\": 150.8760426044464}, \"fc4832696b\": {\"quality\": 0.032692307692307694, \"cost\": 0.001692516, \"time\": 101.06289932727813}, \"fccaadfcdb\": {\"quality\": 0.2098290598290598, \"cost\": 0.008330596999999999, \"time\": 168.6835078954697}, \"fcd3d2b250\": {\"quality\": 0.18482905982905984, \"cost\": 0.008048325, \"time\": 152.26023478507994}, \"fce38334b2\": {\"quality\": 0.21752136752136753, \"cost\": 0.01259885, \"time\": 138.37961835861205}, \"fd0709359e\": {\"quality\": 0.16538461538461538, \"cost\": 0.0006104649999999999, \"time\": 45.40659141540527}, \"fddccfbf94\": {\"quality\": 0.21752136752136753, \"cost\": 0.010504784000000001, \"time\": 108.93906075954438}, \"fe2b4d4d8b\": {\"quality\": 0.24316239316239316, \"cost\": 0.003949674, \"time\": 128.27600796222686}, \"fea4734c09\": {\"quality\": 0.0702991452991453, \"cost\": 0.0056253, \"time\": 73.18559403419495}, \"fef1ca27fa\": {\"quality\": 0.09252136752136753, \"cost\": 0.012806324, \"time\": 116.60554842948915}, \"ff11cb6a7a\": {\"quality\": 0.19252136752136753, \"cost\": 0.012347635999999999, \"time\": 110.44815881252289}, \"ff171e34e2\": {\"quality\": 0.0, \"cost\": 0.0037192830000000003, \"time\": 85.79805471897126}, \"ff1c958e21\": {\"quality\": 0.0, \"cost\": 0.002683884, \"time\": 74.0030293226242}, \"ff8e68049a\": {\"quality\": 0.10085470085470086, \"cost\": 0.006070932, \"time\": 85.60206592082977}}" }, { "path": "abacus-research/biodex-priors.json", "content": "{\"00c93aec22\": {\"quality\": 0.06301075268817204, \"cost\": 0.011813586, \"time\": 87.30672872066498}, \"00f4acd0d3\": {\"quality\": 0.0, \"cost\": 0.0060519300000000005, \"time\": 43.473920822143555}, \"0121878170\": {\"quality\": 0.10301075268817204, \"cost\": 0.039456927, \"time\": 69.10648686885834}, \"01c2f973ad\": {\"quality\": 0.049677419354838714, \"cost\": 0.002479065, \"time\": 42.231227970123285}, \"01fca3c717\": {\"quality\": 0.0929032258064516, \"cost\": 0.029593058000000005, \"time\": 37.2765305519104}, \"02078988c1\": {\"quality\": 0.0, \"cost\": 0.007589632000000001, \"time\": 36.742701053619385}, \"021604dec1\": {\"quality\": 0.10301075268817204, \"cost\": 0.009616495, \"time\": 70.4432460308075}, \"0262668df7\": {\"quality\": 0.10301075268817204, \"cost\": 0.007743969, \"time\": 52.63532865047455}, \"0267c97b70\": {\"quality\": 0.0, \"cost\": 0.0035319149999999996, \"time\": 44.35502307415008}, \"02d6cdecdc\": {\"quality\": 0.10301075268817204, \"cost\": 0.019585369999999998, \"time\": 71.90730912685395}, \"033ca325e6\": {\"quality\": 0.03935483870967742, \"cost\": 0.0024435539999999997, \"time\": 32.108777427673346}, \"0364b5e990\": {\"quality\": 0.06258064516129032, \"cost\": 0.03804064, \"time\": 75.88355865478516}, \"0375ea52c9\": {\"quality\": 0.12301075268817205, \"cost\": 0.011332940000000001, \"time\": 23.44224610328674}, \"038a5f0a62\": {\"quality\": 0.10301075268817204, \"cost\": 0.006738629999999999, \"time\": 72.20387270450593}, \"039803b3b1\": {\"quality\": 0.12301075268817205, \"cost\": 0.029192550000000005, \"time\": 28.46143364906311}, \"03b972cb56\": {\"quality\": 0.10946236559139785, \"cost\": 0.024332390000000002, \"time\": 96.93496084213257}, \"042d933706\": {\"quality\": 0.11591397849462365, \"cost\": 0.008572104, \"time\": 87.77981555461884}, \"050b21ce37\": {\"quality\": 0.10623655913978494, \"cost\": 0.033586281999999995, \"time\": 91.73336846828461}, \"0524f42520\": {\"quality\": 0.10301075268817204, \"cost\": 0.04777091600000001, \"time\": 100.57431967258452}, \"05420351e5\": {\"quality\": 0.10301075268817204, \"cost\": 0.01269706, \"time\": 76.56351339817047}, \"057e332ab1\": {\"quality\": 0.12623655913978493, \"cost\": 0.03724580200000001, \"time\": 96.07672710418701}, \"0646f3f0fb\": {\"quality\": 0.12301075268817205, \"cost\": 0.03542941100000001, \"time\": 98.46862163543702}, \"06493715cc\": {\"quality\": 0.0, \"cost\": 0.005305185, \"time\": 88.10458414554597}, \"0659531b94\": {\"quality\": 0.12623655913978493, \"cost\": 0.031133855000000005, \"time\": 76.578067111969}, \"067ee6e91b\": {\"quality\": 0.10301075268817204, \"cost\": 0.007431704999999999, \"time\": 72.39745917320252}, \"0695f9b5fc\": {\"quality\": 0.020046082949308756, \"cost\": 0.008172494999999998, \"time\": 89.09118111133574}, \"06e94a0f2e\": {\"quality\": 0.10301075268817204, \"cost\": 0.008446952, \"time\": 43.131861782073976}, \"073ef31d23\": {\"quality\": 0.0929032258064516, \"cost\": 0.031532903, \"time\": 66.24685339927674}, \"078a7e545e\": {\"quality\": 0.10301075268817204, \"cost\": 0.031617475000000006, \"time\": 66.09703652858735}, \"079feb14a8\": {\"quality\": 0.11591397849462365, \"cost\": 0.008705902, \"time\": 48.95988223552703}, \"07a3a7daf7\": {\"quality\": 0.0, \"cost\": 0.007065148, \"time\": 43.999537682533266}, \"08127cd6dd\": {\"quality\": 0.11268817204301075, \"cost\": 0.013250458, \"time\": 49.02173194885254}, \"0833133620\": {\"quality\": 0.10623655913978494, \"cost\": 0.00606747, \"time\": 52.81210649013519}, \"087a2cabc4\": {\"quality\": 0.11591397849462365, \"cost\": 0.020192525000000003, \"time\": 94.1397991657257}, \"08bf8cc191\": {\"quality\": 0.0, \"cost\": 0.012546207, \"time\": 107.96688709259033}, \"08e1802287\": {\"quality\": 0.12946236559139784, \"cost\": 0.0034025700000000006, \"time\": 25.23361747264862}, \"08f7f63b30\": {\"quality\": 0.07290322580645162, \"cost\": 0.031714827, \"time\": 104.96963531970978}, \"090cd3ef31\": {\"quality\": 0.10946236559139785, \"cost\": 0.00352441, \"time\": 23.527972793579103}, \"0947216ece\": {\"quality\": 0.11268817204301075, \"cost\": 0.014792812, \"time\": 80.49954354763031}, \"096d51f670\": {\"quality\": 0.08301075268817204, \"cost\": 0.009779318999999998, \"time\": 120.63433356285095}, \"09791c731b\": {\"quality\": 0.08623655913978495, \"cost\": 0.007008072, \"time\": 91.62384450435638}, \"0990c0d4f8\": {\"quality\": 0.10301075268817204, \"cost\": 0.007845877, \"time\": 85.36602926254272}, \"0a4c1bbb4a\": {\"quality\": 0.0, \"cost\": 0.0061861469999999995, \"time\": 90.79566612243653}, \"0ac969dde3\": {\"quality\": 0.10301075268817204, \"cost\": 0.010119635, \"time\": 114.36860978603363}, \"0b4ab72197\": {\"quality\": 0.08301075268817204, \"cost\": 0.010272084, \"time\": 85.533615732193}, \"0bf9d31691\": {\"quality\": 0.0, \"cost\": 0.04230826, \"time\": 110.75088348388672}, \"0c020b86a3\": {\"quality\": 0.06, \"cost\": 0.009179564, \"time\": 80.5141788482666}, \"0c6c7fe96a\": {\"quality\": 0.11591397849462365, \"cost\": 0.012506979000000001, \"time\": 104.92288007736207}, \"0c81c8996a\": {\"quality\": 0.03333333333333333, \"cost\": 0.006368825999999999, \"time\": 67.1365476846695}, \"0cd25da9fe\": {\"quality\": 0.1429032258064516, \"cost\": 0.037754836, \"time\": 88.54509041309356}, \"0cd78f33d8\": {\"quality\": 0.12301075268817205, \"cost\": 0.05730706500000001, \"time\": 70.51942982673646}, \"0cdc5954dd\": {\"quality\": 0.06623655913978495, \"cost\": 0.008238566000000001, \"time\": 70.26068692207338}, \"0d53dd53c1\": {\"quality\": 0.10301075268817204, \"cost\": 0.040789094, \"time\": 83.40459337234498}, \"0d8436af32\": {\"quality\": 0.10301075268817204, \"cost\": 0.011668765, \"time\": 70.86097333431243}, \"0e38896654\": {\"quality\": 0.12301075268817205, \"cost\": 0.011624445, \"time\": 69.54783916473389}, \"0ec672e7c8\": {\"quality\": 0.10301075268817204, \"cost\": 0.013338835, \"time\": 92.46337025165558}, \"0ed243f788\": {\"quality\": 0.11591397849462365, \"cost\": 0.009052659000000001, \"time\": 82.04982092380524}, \"0eeb372802\": {\"quality\": 0.08489247311827958, \"cost\": 0.01393459, \"time\": 87.2686581134796}, \"0ef0becc1b\": {\"quality\": 0.14562980030721967, \"cost\": 0.028691548000000004, \"time\": 57.96495416164398}, \"0fefead197\": {\"quality\": 0.10623655913978494, \"cost\": 0.036967563, \"time\": 86.80253887176514}, \"0ff126ebf8\": {\"quality\": 0.10623655913978494, \"cost\": 0.012027738000000001, \"time\": 62.63344478607178}, \"10d1d4bdeb\": {\"quality\": 0.0, \"cost\": 0.034578758, \"time\": 84.60008835792542}, \"114a097c53\": {\"quality\": 0.02967741935483871, \"cost\": 0.00546162, \"time\": 32.99331085681915}, \"1175ee37e6\": {\"quality\": 0.06301075268817204, \"cost\": 0.0015683849999999998, \"time\": 27.241142559051514}, \"11bc996d48\": {\"quality\": 0.10301075268817204, \"cost\": 0.006689505, \"time\": 67.14279909133911}, \"11ded03305\": {\"quality\": 0.12301075268817205, \"cost\": 0.03211208700000001, \"time\": 71.28769545555114}, \"123fb650fb\": {\"quality\": 0.0704147465437788, \"cost\": 0.009157566, \"time\": 94.18252191543579}, \"132f6f3946\": {\"quality\": 0.0, \"cost\": 0.034120617000000006, \"time\": 80.40761358737946}, \"133ee5023f\": {\"quality\": 0.08301075268817204, \"cost\": 0.0056809600000000005, \"time\": 41.6411509513855}, \"13a009fe0c\": {\"quality\": 0.12290322580645162, \"cost\": 0.013064816, \"time\": 103.28004715442657}, \"13da306f84\": {\"quality\": 0.10301075268817204, \"cost\": 0.009303667999999998, \"time\": 75.9394079208374}, \"13e717e41e\": {\"quality\": 0.10301075268817204, \"cost\": 0.027711322000000004, \"time\": 46.58983483314514}, \"13f75f9bd0\": {\"quality\": 0.06301075268817204, \"cost\": 0.007269792000000001, \"time\": 78.83892266750337}, \"140ededb41\": {\"quality\": 0.07290322580645162, \"cost\": 0.0016495350000000002, \"time\": 27.533066105842593}, \"142e59c03f\": {\"quality\": 0.10301075268817204, \"cost\": 0.051444216, \"time\": 89.78037304878235}, \"142f3a7c70\": {\"quality\": 0.0, \"cost\": 0.0048542970000000005, \"time\": 90.07974328994752}, \"1468dddecc\": {\"quality\": 0.12623655913978493, \"cost\": 0.0025156650000000003, \"time\": 58.17216382026672}, \"15af009a01\": {\"quality\": 0.09333333333333334, \"cost\": 0.03807909200000001, \"time\": 106.9638382911682}, \"15b80a55d3\": {\"quality\": 0.09333333333333334, \"cost\": 0.036063054, \"time\": 102.59992785453797}, \"1625e624c5\": {\"quality\": 0.0696774193548387, \"cost\": 0.02807447, \"time\": 56.73278093338013}, \"16cff1c1e9\": {\"quality\": 0.0, \"cost\": 0.031882792, \"time\": 110.76414783000945}, \"17407df027\": {\"quality\": 0.08623655913978495, \"cost\": 0.035592325999999994, \"time\": 101.71423263549804}, \"176da24f53\": {\"quality\": 0.014285714285714285, \"cost\": 0.004459658, \"time\": 50.037702202796936}, \"179379555f\": {\"quality\": 0.08946236559139785, \"cost\": 0.006388011, \"time\": 77.25874888896942}, \"181c91d1be\": {\"quality\": 0.10623655913978494, \"cost\": 0.012226980000000002, \"time\": 70.32721478939057}, \"183743e76e\": {\"quality\": 0.0, \"cost\": 0.00601107, \"time\": 54.00228130817413}, \"186b58c209\": {\"quality\": 0.10623655913978494, \"cost\": 0.03340172, \"time\": 77.13717110157013}, \"187eace9fe\": {\"quality\": 0.0, \"cost\": 0.031072944, \"time\": 83.48713932037353}, \"190ed2e1b6\": {\"quality\": 0.20301075268817204, \"cost\": 0.032216430000000004, \"time\": 60.7668753862381}, \"191aafe1a6\": {\"quality\": 0.10301075268817204, \"cost\": 0.028626901, \"time\": 69.7408289194107}, \"194919ad28\": {\"quality\": 0.032903225806451615, \"cost\": 0.032168366000000004, \"time\": 64.83868379592896}, \"197bb53f10\": {\"quality\": 0.09333333333333334, \"cost\": 0.0064794810000000005, \"time\": 85.31207880973815}, \"19ba7d0617\": {\"quality\": 0.12301075268817205, \"cost\": 0.0850125, \"time\": 65.46971595287323}, \"19e3db7fe7\": {\"quality\": 0.03333333333333333, \"cost\": 0.012209139999999999, \"time\": 111.22203342914581}, \"1a08cb3f50\": {\"quality\": 0.0, \"cost\": 0.031574925000000004, \"time\": 48.5790287733078}, \"1a169179f6\": {\"quality\": 0.10301075268817204, \"cost\": 0.044995354, \"time\": 103.08906679153444}, \"1a71d61ac4\": {\"quality\": 0.08301075268817204, \"cost\": 0.02794963, \"time\": 76.25873317718506}, \"1ad856985f\": {\"quality\": 0.02258064516129032, \"cost\": 0.009150617, \"time\": 122.38001575469971}, \"1adec2dca2\": {\"quality\": 0.0, \"cost\": 0.01436658, \"time\": 146.7017418861389}, \"1b04a2b184\": {\"quality\": 0.11591397849462365, \"cost\": 0.013684314999999999, \"time\": 133.7267296552658}, \"1b28439bd7\": {\"quality\": 0.21591397849462365, \"cost\": 0.011476702000000002, \"time\": 100.34343218803406}, \"1b2c667b15\": {\"quality\": 0.19333333333333333, \"cost\": 0.033035388, \"time\": 89.06020038127899}, \"1b4511eada\": {\"quality\": 0.11456989247311827, \"cost\": 0.030632538, \"time\": 129.76828122138977}, \"1b7e6cad66\": {\"quality\": 0.10301075268817204, \"cost\": 0.028176118, \"time\": 68.36105287075043}, \"1beb2fac62\": {\"quality\": 0.10301075268817204, \"cost\": 0.013600296000000001, \"time\": 84.02366802692413}, \"1c347e4d91\": {\"quality\": 0.10301075268817204, \"cost\": 0.003799887, \"time\": 104.80383982658387}, \"1c35bf4be6\": {\"quality\": 0.10623655913978494, \"cost\": 0.03395996, \"time\": 116.73827676773071}, \"1c4bbf8f7e\": {\"quality\": 0.10301075268817204, \"cost\": 0.04330549100000001, \"time\": 104.88216242790222}, \"1c5f1341f6\": {\"quality\": 0.10623655913978494, \"cost\": 0.023331052, \"time\": 78.7630702495575}, \"1c71804bec\": {\"quality\": 0.08967741935483872, \"cost\": 0.019993577999999998, \"time\": 79.78316872119903}, \"1cc6d9efb6\": {\"quality\": 0.10301075268817204, \"cost\": 0.012043020000000002, \"time\": 78.92661263942719}, \"1ce3d77039\": {\"quality\": 0.10301075268817204, \"cost\": 0.000653472, \"time\": 41.55318109989166}, \"1d26090364\": {\"quality\": 0.12301075268817205, \"cost\": 0.004974435, \"time\": 56.773493957519534}, \"1d87f97e62\": {\"quality\": 0.10301075268817204, \"cost\": 0.007740711000000001, \"time\": 77.09953954219819}, \"1d90fb8ca6\": {\"quality\": 0.10623655913978494, \"cost\": 0.034693865000000004, \"time\": 72.57712695598602}, \"1da2369719\": {\"quality\": 0.032903225806451615, \"cost\": 0.004285659000000001, \"time\": 71.20325164794922}, \"1e18e60895\": {\"quality\": 0.12301075268817205, \"cost\": 0.0006631269999999999, \"time\": 39.11586356163025}, \"1e1bf7e88b\": {\"quality\": 0.10623655913978494, \"cost\": 0.012434028, \"time\": 72.40708291530609}, \"1e8b3521f8\": {\"quality\": 0.07729646697388634, \"cost\": 0.003250434, \"time\": 60.45711851119995}, \"1f412964ff\": {\"quality\": 0.12301075268817205, \"cost\": 0.029133057000000004, \"time\": 64.97693083286285}, \"1f72cfb78a\": {\"quality\": 0.21591397849462365, \"cost\": 0.020125592, \"time\": 99.64003388881683}, \"1fb5d170ad\": {\"quality\": 0.11268817204301075, \"cost\": 0.031407344000000004, \"time\": 107.89154133796691}, \"20180dd292\": {\"quality\": 0.03333333333333333, \"cost\": 0.035306141, \"time\": 115.35444395542144}, \"2018bef45f\": {\"quality\": 0.10301075268817204, \"cost\": 0.007217930000000001, \"time\": 68.45855422019959}, \"2075ff1d04\": {\"quality\": 0.06, \"cost\": 0.000997618, \"time\": 56.56468660831451}, \"208a98f514\": {\"quality\": 0.06623655913978495, \"cost\": 0.010588595, \"time\": 121.87733623981475}, \"20904e5c14\": {\"quality\": 0.08967741935483872, \"cost\": 0.031855074, \"time\": 83.47015318870544}, \"20afc3d539\": {\"quality\": 0.23591397849462367, \"cost\": 0.039295448, \"time\": 91.55873837471009}, \"20e10af7d4\": {\"quality\": 0.09591397849462366, \"cost\": 0.005751711, \"time\": 109.45092914104461}, \"20e2c0b057\": {\"quality\": 0.04, \"cost\": 0.008348217000000002, \"time\": 118.59764387607575}, \"211b89b4cd\": {\"quality\": 0.10301075268817204, \"cost\": 0.005769704999999999, \"time\": 68.7529235124588}, \"21386082aa\": {\"quality\": 0.10946236559139785, \"cost\": 0.019664704, \"time\": 109.10838623046875}, \"21b2b8ebd1\": {\"quality\": 0.10623655913978494, \"cost\": 0.0073147049999999995, \"time\": 87.6013917684555}, \"21b78249a7\": {\"quality\": 0.10301075268817204, \"cost\": 0.04023389200000001, \"time\": 85.05358052253723}, \"21bed16a7d\": {\"quality\": 0.05290322580645161, \"cost\": 0.0014240299999999997, \"time\": 44.216261696815494}, \"2200d969d0\": {\"quality\": 0.10301075268817204, \"cost\": 0.042653065000000004, \"time\": 104.29301726818085}, \"220d008704\": {\"quality\": 0.02, \"cost\": 0.030364681, \"time\": 107.97830784320831}, \"2251d21392\": {\"quality\": 0.11591397849462365, \"cost\": 0.033143465, \"time\": 79.32752299308777}, \"23566f15ab\": {\"quality\": 0.08301075268817204, \"cost\": 0.0005060469999999999, \"time\": 43.435147762298584}, \"23a9506d36\": {\"quality\": 0.08301075268817204, \"cost\": 0.005874952, \"time\": 36.02558331489563}, \"24957f3a43\": {\"quality\": 0.08946236559139785, \"cost\": 0.004644396, \"time\": 53.15490672588348}, \"2529e2f8b0\": {\"quality\": 0.10301075268817204, \"cost\": 0.007180190000000001, \"time\": 28.930508136749268}, \"252f01ac5b\": {\"quality\": 0.12623655913978493, \"cost\": 0.04125042200000001, \"time\": 92.3930985212326}, \"25fadf0883\": {\"quality\": 0.10946236559139785, \"cost\": 0.03512302, \"time\": 116.36114256381988}, \"2609bfd616\": {\"quality\": 0.10946236559139785, \"cost\": 0.009596338, \"time\": 116.62334115505219}, \"2629f3e324\": {\"quality\": 0.12301075268817205, \"cost\": 0.033418977, \"time\": 125.81296608448028}, \"262e4298f9\": {\"quality\": 0.06301075268817204, \"cost\": 0.03408896, \"time\": 126.59840724468232}, \"26cc40d3bb\": {\"quality\": 0.06258064516129032, \"cost\": 0.035955086, \"time\": 118.60315663814545}, \"2728c8eb6a\": {\"quality\": 0.063963133640553, \"cost\": 0.008921946, \"time\": 100.46461434364319}, \"27bc52befa\": {\"quality\": 0.08623655913978495, \"cost\": 0.0216924, \"time\": 65.37463784217834}, \"27daa50458\": {\"quality\": 0.19978494623655915, \"cost\": 0.005977634, \"time\": 62.87260589599609}, \"2821795e69\": {\"quality\": 0.11591397849462365, \"cost\": 0.035850537, \"time\": 122.94275135993958}, \"28369b2421\": {\"quality\": 0.11591397849462365, \"cost\": 0.008891583000000002, \"time\": 96.52785930633544}, \"28421e6d62\": {\"quality\": 0.07967741935483871, \"cost\": 0.007445298000000001, \"time\": 90.94689693450928}, \"2936c3e43e\": {\"quality\": 0.10623655913978494, \"cost\": 0.012975128999999998, \"time\": 113.21974639892579}, \"293ec5edca\": {\"quality\": 0.10301075268817204, \"cost\": 0.006139281, \"time\": 76.99833080768585}, \"29409d0894\": {\"quality\": 0.08623655913978495, \"cost\": 0.005303836, \"time\": 53.642726635932924}, \"294258298a\": {\"quality\": 0.08301075268817204, \"cost\": 0.029248269, \"time\": 73.05858094692229}, \"294e541235\": {\"quality\": 0.014285714285714285, \"cost\": 0.002588631, \"time\": 46.93031387329101}, \"295ed5e759\": {\"quality\": 0.03333333333333333, \"cost\": 0.007936986, \"time\": 78.30404438972474}, \"2960431101\": {\"quality\": 0.07913978494623655, \"cost\": 0.007295059999999999, \"time\": 65.45906202793121}, \"29892d8468\": {\"quality\": 0.0, \"cost\": 0.012367010000000001, \"time\": 107.3028032541275}, \"299a0aeb65\": {\"quality\": 0.10301075268817204, \"cost\": 0.008972781999999999, \"time\": 99.65134017467498}, \"29ad99e3ed\": {\"quality\": 0.012903225806451613, \"cost\": 0.028833116000000002, \"time\": 76.06020951271057}, \"2a5edac2de\": {\"quality\": 0.10301075268817204, \"cost\": 0.040388614, \"time\": 98.854678440094}, \"2a7d15f4a7\": {\"quality\": 0.10301075268817204, \"cost\": 0.0007995049999999998, \"time\": 26.340464663505553}, \"2aa996de6a\": {\"quality\": 0.13591397849462367, \"cost\": 0.041571458000000006, \"time\": 101.75389010906218}, \"2ac4fb293f\": {\"quality\": 0.10301075268817204, \"cost\": 0.027978918000000002, \"time\": 79.7981543302536}, \"2afeff0083\": {\"quality\": 0.13134408602150538, \"cost\": 0.034197613, \"time\": 108.6436208486557}, \"2b2bc9568b\": {\"quality\": 0.12301075268817205, \"cost\": 0.034515569, \"time\": 122.13092226982117}, \"2b5679d248\": {\"quality\": 0.07591397849462365, \"cost\": 0.012507200999999999, \"time\": 111.41059238910675}, \"2bcf54cda1\": {\"quality\": 0.05333333333333333, \"cost\": 0.03352914, \"time\": 122.20578529834748}, \"2bd39ee744\": {\"quality\": 0.10301075268817204, \"cost\": 0.016464735, \"time\": 93.0221663236618}, \"2bf38d797f\": {\"quality\": 0.09655913978494624, \"cost\": 0.007761932999999999, \"time\": 85.16239807605743}, \"2c4f4f304e\": {\"quality\": 0.12301075268817205, \"cost\": 0.004792675, \"time\": 50.257468938827515}, \"2c5cf9eb26\": {\"quality\": 0.0, \"cost\": 0.006811359, \"time\": 85.30586485862732}, \"2c9a9f94c4\": {\"quality\": 0.06623655913978495, \"cost\": 0.011060496, \"time\": 127.40647723674773}, \"2d3bbc2d23\": {\"quality\": 0.11333333333333334, \"cost\": 0.006915585, \"time\": 98.87370185852052}, \"2de113167b\": {\"quality\": 0.20623655913978495, \"cost\": 0.0120975, \"time\": 114.90267214775085}, \"2de3eb2c19\": {\"quality\": 0.08258064516129032, \"cost\": 0.006905800000000001, \"time\": 56.68419787883758}, \"2e30394ac6\": {\"quality\": 0.09333333333333334, \"cost\": 0.007497251999999999, \"time\": 99.04903969764709}, \"2e9c5cc9bf\": {\"quality\": 0.05290322580645161, \"cost\": 0.006026862, \"time\": 64.40614473819733}, \"2f1573da80\": {\"quality\": 0.0696774193548387, \"cost\": 0.009294684000000001, \"time\": 118.76816306114196}, \"2f39d78f34\": {\"quality\": 0.12623655913978493, \"cost\": 0.03424951, \"time\": 84.87492218017579}, \"2fc0cb3592\": {\"quality\": 0.07290322580645162, \"cost\": 0.007008576, \"time\": 93.76994898319245}, \"2fd9cd426a\": {\"quality\": 0.08301075268817204, \"cost\": 0.012578212000000002, \"time\": 125.21965701580046}, \"300924ebae\": {\"quality\": 0.10301075268817204, \"cost\": 0.034383670000000005, \"time\": 79.05412302017211}, \"3019af79b3\": {\"quality\": 0.05333333333333333, \"cost\": 0.000728678, \"time\": 50.62756032943726}, \"302c1d97fc\": {\"quality\": 0.032903225806451615, \"cost\": 0.012643149999999999, \"time\": 94.44454934597016}, \"30ae4cbe91\": {\"quality\": 0.04967741935483871, \"cost\": 0.0006998079999999999, \"time\": 55.2713984966278}, \"30c1f9ddf1\": {\"quality\": 0.02, \"cost\": 0.031684431000000006, \"time\": 81.2769385099411}, \"30cd375570\": {\"quality\": 0.10623655913978494, \"cost\": 0.034244302000000004, \"time\": 84.5775047302246}, \"3169782cbb\": {\"quality\": 0.12623655913978493, \"cost\": 0.030658732000000005, \"time\": 82.21524584293365}, \"3172fc459a\": {\"quality\": 0.20623655913978495, \"cost\": 0.039465366, \"time\": 106.1840931892395}, \"318499c14b\": {\"quality\": 0.06623655913978495, \"cost\": 0.03394029, \"time\": 109.55087904930114}, \"31a32be94d\": {\"quality\": 0.12623655913978493, \"cost\": 0.040973235, \"time\": 113.42904851436614}, \"32b101d807\": {\"quality\": 0.11134408602150538, \"cost\": 0.013450305, \"time\": 116.4323141336441}, \"32e2c7ad7f\": {\"quality\": 0.10623655913978494, \"cost\": 0.034506176, \"time\": 105.92563235759735}, \"33459cd29c\": {\"quality\": 0.05333333333333333, \"cost\": 0.009939888, \"time\": 108.05581707954407}, \"33a187e74f\": {\"quality\": 0.10301075268817204, \"cost\": 0.00615336, \"time\": 108.04435174465179}, \"33bab4f766\": {\"quality\": 0.10946236559139785, \"cost\": 0.030359016, \"time\": 94.6774396419525}, \"34922140da\": {\"quality\": 0.12623655913978493, \"cost\": 0.037889484, \"time\": 97.54083952903747}, \"3511b5e1d0\": {\"quality\": 0.0, \"cost\": 0.009048384, \"time\": 107.80388951301575}, \"3513311c2d\": {\"quality\": 0.10946236559139785, \"cost\": 0.030197856000000002, \"time\": 70.30446681976318}, \"3513e54767\": {\"quality\": 0.1529032258064516, \"cost\": 0.006643148000000001, \"time\": 51.87993569374085}, \"353f0cb1ac\": {\"quality\": 0.07591397849462365, \"cost\": 0.002595798, \"time\": 43.97162518501281}, \"3550bf88cb\": {\"quality\": 0.10301075268817204, \"cost\": 0.008131596, \"time\": 101.78304505348206}, \"35610fb420\": {\"quality\": 0.10301075268817204, \"cost\": 0.004529697, \"time\": 86.15702996253967}, \"357267e14b\": {\"quality\": 0.10301075268817204, \"cost\": 0.001159102, \"time\": 43.801218032836914}, \"35baa5c3cc\": {\"quality\": 0.12623655913978493, \"cost\": 0.032426465, \"time\": 97.53744850158691}, \"3637084f91\": {\"quality\": 0.11913978494623656, \"cost\": 0.00731297, \"time\": 75.00282621383667}, \"368a497102\": {\"quality\": 0.08946236559139785, \"cost\": 0.023536822000000006, \"time\": 74.35990719795228}, \"36c66671ee\": {\"quality\": 0.049677419354838714, \"cost\": 0.0066839249999999985, \"time\": 80.50561456680299}, \"37456cb002\": {\"quality\": 0.10623655913978494, \"cost\": 0.018729613, \"time\": 104.19728450775146}, \"3746ea5c03\": {\"quality\": 0.08946236559139785, \"cost\": 0.025954912000000004, \"time\": 77.87212231159211}, \"375ed248fe\": {\"quality\": 0.09333333333333334, \"cost\": 0.009719407999999999, \"time\": 83.5635404586792}, \"377cdf9209\": {\"quality\": 0.10301075268817204, \"cost\": 0.015890171, \"time\": 106.57615406513214}, \"37d4d0f214\": {\"quality\": 0.09913978494623656, \"cost\": 0.035574514, \"time\": 106.92880241870881}, \"37ece7217f\": {\"quality\": 0.11729646697388633, \"cost\": 0.027270672000000003, \"time\": 49.32679312229156}, \"38075bb01f\": {\"quality\": 0.05333333333333333, \"cost\": 0.0043161810000000005, \"time\": 80.82736201286316}, \"3831d758b1\": {\"quality\": 0.0, \"cost\": 0.032444862000000005, \"time\": 82.06478433609009}, \"38567d6a43\": {\"quality\": 0.0, \"cost\": 0.004685955, \"time\": 89.93808691501617}, \"3875787727\": {\"quality\": 0.08301075268817204, \"cost\": 0.034211713000000005, \"time\": 105.70414273738861}, \"389c54cbca\": {\"quality\": 0.09655913978494624, \"cost\": 0.00570144, \"time\": 52.78039865493774}, \"3980f20caa\": {\"quality\": 0.05290322580645161, \"cost\": 0.014630126000000002, \"time\": 109.6514874458313}, \"3997a836bd\": {\"quality\": 0.10301075268817204, \"cost\": 0.034291603000000004, \"time\": 108.07982413768768}, \"39ad76f8ce\": {\"quality\": 0.10301075268817204, \"cost\": 0.033542926, \"time\": 88.07275912761688}, \"39c0b7c171\": {\"quality\": 0.10301075268817204, \"cost\": 0.036100823000000004, \"time\": 108.34460818767548}, \"39cd4ca402\": {\"quality\": 0.02, \"cost\": 0.0034239629999999995, \"time\": 84.85748331546785}, \"3a32c98a53\": {\"quality\": 0.10623655913978494, \"cost\": 0.030208363000000002, \"time\": 82.875235414505}, \"3ac7fa4e46\": {\"quality\": 0.10301075268817204, \"cost\": 0.04524012, \"time\": 97.64954562187195}, \"3ad6dcf559\": {\"quality\": 0.0, \"cost\": 0.031851822, \"time\": 80.59675960540771}, \"3ae0de8663\": {\"quality\": 0.10301075268817204, \"cost\": 0.021463644000000004, \"time\": 97.35158727169036}, \"3b2e8075ea\": {\"quality\": 0.08623655913978495, \"cost\": 0.003255993, \"time\": 67.06967768669128}, \"3b3676521a\": {\"quality\": 0.10301075268817204, \"cost\": 0.013791060999999999, \"time\": 95.51536769866944}, \"3b57530a56\": {\"quality\": 0.10301075268817204, \"cost\": 0.002789373, \"time\": 44.8931683063507}, \"3b6fbfa11d\": {\"quality\": 0.10946236559139785, \"cost\": 0.016405658, \"time\": 70.66574103832244}, \"3b81215e7a\": {\"quality\": 0.11268817204301075, \"cost\": 0.034937565000000004, \"time\": 111.87461152076722}, \"3b9f8045d7\": {\"quality\": 0.11268817204301075, \"cost\": 0.044838478, \"time\": 104.71198470592498}, \"3c5857683c\": {\"quality\": 0.10946236559139785, \"cost\": 0.03896547, \"time\": 133.13998155593873}, \"3cbab8082e\": {\"quality\": 0.09333333333333334, \"cost\": 0.009450357, \"time\": 131.19907307624817}, \"3d21104666\": {\"quality\": 0.12301075268817205, \"cost\": 0.05427800000000001, \"time\": 58.033410573005675}, \"3d71c4dd2c\": {\"quality\": 0.10301075268817204, \"cost\": 0.006745917, \"time\": 99.85713205337524}, \"3d9e24215e\": {\"quality\": 0.10301075268817204, \"cost\": 0.008923918, \"time\": 54.70108168125152}, \"3e7efee65a\": {\"quality\": 0.09268817204301075, \"cost\": 0.034440120000000005, \"time\": 125.9171290397644}, \"3ed0ad20ed\": {\"quality\": 0.12301075268817205, \"cost\": 0.038282694000000006, \"time\": 132.21433353424072}, \"3f1a58aec9\": {\"quality\": 0.22301075268817205, \"cost\": 0.0019911300000000002, \"time\": 48.89557406902313}, \"3f2b07cb78\": {\"quality\": 0.10946236559139785, \"cost\": 0.016589166000000002, \"time\": 105.48563659191132}, \"3f3ef494b0\": {\"quality\": 0.0, \"cost\": 0.001254132, \"time\": 57.60023159980774}, \"3f62c3fbfc\": {\"quality\": 0.0696774193548387, \"cost\": 0.0044856359999999994, \"time\": 95.2896116256714}, \"3f730d8bfe\": {\"quality\": 0.10301075268817204, \"cost\": 0.04548097400000001, \"time\": 94.35254197120668}, \"3f8d2ee81f\": {\"quality\": 0.10623655913978494, \"cost\": 0.02839726, \"time\": 88.55384962558746}, \"40104c813f\": {\"quality\": 0.00967741935483871, \"cost\": 0.010927247000000001, \"time\": 137.43559978008273}, \"403b05da2d\": {\"quality\": 0.02, \"cost\": 0.010782219999999999, \"time\": 89.91662650108339}, \"403f0726fa\": {\"quality\": 0.10301075268817204, \"cost\": 0.030024500000000003, \"time\": 53.93264591693878}, \"4098178354\": {\"quality\": 0.10623655913978494, \"cost\": 0.033845859000000006, \"time\": 113.72850313186646}, \"409ff67607\": {\"quality\": 0.06, \"cost\": 0.004197711, \"time\": 99.27511491775513}, \"40b3b6642c\": {\"quality\": 0.12301075268817205, \"cost\": 0.033175452, \"time\": 78.3308812379837}, \"412c065b83\": {\"quality\": 0.03333333333333333, \"cost\": 0.00797817, \"time\": 76.86819369792937}, \"4191118787\": {\"quality\": 0.10301075268817204, \"cost\": 0.009176240999999998, \"time\": 115.17749345302582}, \"41d5b97871\": {\"quality\": 0.0, \"cost\": 0.002632212, \"time\": 67.86801817417145}, \"41d8845655\": {\"quality\": 0.12301075268817205, \"cost\": 0.011058695, \"time\": 66.63209574222564}, \"41ee202cac\": {\"quality\": 0.04, \"cost\": 0.0015582279999999996, \"time\": 81.97960751056671}, \"41fe4aee55\": {\"quality\": 0.04, \"cost\": 0.033214178, \"time\": 89.46116530895233}, \"42430ea391\": {\"quality\": 0.10301075268817204, \"cost\": 0.037945212000000006, \"time\": 121.86067166328431}, \"42ddd48341\": {\"quality\": 0.10301075268817204, \"cost\": 0.015020527999999998, \"time\": 114.5060165643692}, \"42f1e19aa7\": {\"quality\": 0.11591397849462365, \"cost\": 0.04674747600000001, \"time\": 126.75622403621674}, \"430a2ab32f\": {\"quality\": 0.12301075268817205, \"cost\": 0.039291168, \"time\": 96.79555187225341}, \"4339427ad8\": {\"quality\": 0.08634408602150537, \"cost\": 0.03448633, \"time\": 124.03267226219177}, \"4361bc7ea7\": {\"quality\": 0.07333333333333333, \"cost\": 0.001487792, \"time\": 83.36836609840392}, \"43c3cf9cb8\": {\"quality\": 0.10301075268817204, \"cost\": 0.006907384000000001, \"time\": 57.54507291316986}, \"43d24fb32a\": {\"quality\": 0.10946236559139785, \"cost\": 0.010483464, \"time\": 98.36521954536438}, \"43e9b39e5c\": {\"quality\": 0.12946236559139784, \"cost\": 0.036367179, \"time\": 118.90290827751159}, \"44d6af5523\": {\"quality\": 0.10301075268817204, \"cost\": 0.0051817799999999995, \"time\": 97.96241414546967}, \"44f189d813\": {\"quality\": 0.10623655913978494, \"cost\": 0.020515107000000005, \"time\": 121.6331589460373}, \"450f45a187\": {\"quality\": 0.0, \"cost\": 0.007900044000000002, \"time\": 60.44815545082092}, \"453d0a5097\": {\"quality\": 0.06301075268817204, \"cost\": 0.016095624, \"time\": 92.88813369274139}, \"4547ef4c8e\": {\"quality\": 0.10301075268817204, \"cost\": 0.033263573000000005, \"time\": 89.95854828357696}, \"461846a52d\": {\"quality\": 0.10623655913978494, \"cost\": 0.003919338, \"time\": 85.1272670507431}, \"462e6ff849\": {\"quality\": 0.06761904761904762, \"cost\": 0.011236148000000001, \"time\": 113.67141807079315}, \"4630853d32\": {\"quality\": 0.08946236559139785, \"cost\": 0.008533141000000001, \"time\": 86.52934875488282}, \"46475b9e75\": {\"quality\": 0.10301075268817204, \"cost\": 0.006695604000000001, \"time\": 94.7075261592865}, \"46654a1f32\": {\"quality\": 0.0, \"cost\": 0.012072104, \"time\": 64.53883934020996}, \"466a3036b2\": {\"quality\": 0.06301075268817204, \"cost\": 0.010685084999999999, \"time\": 106.71569502353668}, \"466d4d16dd\": {\"quality\": 0.08946236559139785, \"cost\": 0.008533694000000001, \"time\": 79.9828891992569}, \"46ed68152d\": {\"quality\": 0.07612903225806453, \"cost\": 0.012683225999999999, \"time\": 86.68546965122223}, \"476a12876c\": {\"quality\": 0.08946236559139785, \"cost\": 0.0062402939999999995, \"time\": 72.02064683437348}, \"4778401a7a\": {\"quality\": 0.10301075268817204, \"cost\": 0.016234258, \"time\": 93.26795687675477}, \"47f9115b26\": {\"quality\": 0.03333333333333333, \"cost\": 0.03462185200000001, \"time\": 101.22674057483673}, \"48043e2304\": {\"quality\": 0.10301075268817204, \"cost\": 0.009490545, \"time\": 83.147008061409}, \"487f30e740\": {\"quality\": 0.10623655913978494, \"cost\": 0.030731074000000004, \"time\": 72.48842267990112}, \"488645cbd9\": {\"quality\": 0.07290322580645162, \"cost\": 0.0015898319999999998, \"time\": 47.430080437660216}, \"48bf87f7fe\": {\"quality\": 0.11591397849462365, \"cost\": 0.041125237999999995, \"time\": 95.13780512809754}, \"4909061216\": {\"quality\": 0.08301075268817204, \"cost\": 0.013231131, \"time\": 110.22792162895203}, \"49731b1ccd\": {\"quality\": 0.08301075268817204, \"cost\": 0.007518614999999999, \"time\": 77.45786833763123}, \"49ad844bd2\": {\"quality\": 0.08301075268817204, \"cost\": 0.009246200999999999, \"time\": 125.66613755226135}, \"49ca727e49\": {\"quality\": 0.10301075268817204, \"cost\": 0.000910932, \"time\": 36.98862104415893}, \"4a23d8eff7\": {\"quality\": 0.012903225806451613, \"cost\": 0.033723964, \"time\": 133.58634560108186}, \"4a555da784\": {\"quality\": 0.12301075268817205, \"cost\": 0.030253567000000002, \"time\": 93.47855880260468}, \"4a5cea8b85\": {\"quality\": 0.10301075268817204, \"cost\": 0.037096130000000005, \"time\": 87.66496036052703}, \"4a767339bd\": {\"quality\": 0.10946236559139785, \"cost\": 0.034482308, \"time\": 119.38791158199311}, \"4aafd39d76\": {\"quality\": 0.0, \"cost\": 0.005941872, \"time\": 106.78513460159303}, \"4aca6e5216\": {\"quality\": 0.08623655913978495, \"cost\": 0.011614728, \"time\": 98.73750612735748}, \"4b18a647d6\": {\"quality\": 0.10623655913978494, \"cost\": 0.04434360500000001, \"time\": 133.19987111091615}, \"4bc4528402\": {\"quality\": 0.10623655913978494, \"cost\": 0.004866044999999999, \"time\": 105.87921042442322}, \"4c158a1a4a\": {\"quality\": 0.06301075268817204, \"cost\": 0.008219039999999999, \"time\": 107.04837877750397}, \"4c954323e3\": {\"quality\": 0.11591397849462365, \"cost\": 0.010109752999999999, \"time\": 127.91732285022735}, \"4d91e8a27b\": {\"quality\": 0.1761290322580645, \"cost\": 0.010757003000000001, \"time\": 101.67963089942933}, \"4dc185389a\": {\"quality\": 0.08290322580645162, \"cost\": 0.030485803000000006, \"time\": 113.32033824920654}, \"4dd3635bc3\": {\"quality\": 0.12623655913978493, \"cost\": 0.013448202, \"time\": 89.75064220428467}, \"4dd98ef398\": {\"quality\": 0.0, \"cost\": 0.030016080000000004, \"time\": 69.32772116661073}, \"4dfacd0007\": {\"quality\": 0.10623655913978494, \"cost\": 0.014845295, \"time\": 100.57037975788117}, \"4e298ee0d4\": {\"quality\": 0.07612903225806453, \"cost\": 0.008624916, \"time\": 97.4965528011322}, \"4e3443a0f9\": {\"quality\": 0.11591397849462365, \"cost\": 0.013369873000000001, \"time\": 110.12258355617523}, \"4e4b9db2b8\": {\"quality\": 0.116605222734255, \"cost\": 0.007327941000000001, \"time\": 75.85998649597168}, \"4e6509f614\": {\"quality\": 0.08301075268817204, \"cost\": 0.00546564, \"time\": 51.5486006975174}, \"4e6a83e751\": {\"quality\": 0.07290322580645162, \"cost\": 0.000857376, \"time\": 45.70523474216461}, \"4e79c8947f\": {\"quality\": 0.11134408602150539, \"cost\": 0.00732088, \"time\": 43.410219454765326}, \"4e9504432b\": {\"quality\": 0.10623655913978494, \"cost\": 0.034706275, \"time\": 129.37581310272216}, \"4e962170dc\": {\"quality\": 0.02, \"cost\": 0.033169164, \"time\": 83.41308994293212}, \"4eb0826f21\": {\"quality\": 0.0, \"cost\": 0.027521320000000002, \"time\": 56.27700278759002}, \"4ed41bf2e4\": {\"quality\": 0.12623655913978493, \"cost\": 0.03861573, \"time\": 115.21599047183992}, \"4f78672528\": {\"quality\": 0.08301075268817204, \"cost\": 0.037576234, \"time\": 94.65013363361359}, \"4f8cca1195\": {\"quality\": 0.13019969278033794, \"cost\": 0.010837381, \"time\": 124.13806178569794}, \"500860eaa2\": {\"quality\": 0.0, \"cost\": 0.002532492, \"time\": 48.11276173591614}, \"50701b505e\": {\"quality\": 0.09333333333333332, \"cost\": 0.006791684999999999, \"time\": 99.08809175491334}, \"51583a901c\": {\"quality\": 0.14946236559139786, \"cost\": 0.010773869, \"time\": 124.03754835128784}, \"51aeaf9f3e\": {\"quality\": 0.10301075268817204, \"cost\": 0.008776062000000001, \"time\": 40.02036185264588}, \"520b52b64c\": {\"quality\": 0.10301075268817204, \"cost\": 0.0302065, \"time\": 60.96432254314423}, \"521314dab6\": {\"quality\": 0.20301075268817204, \"cost\": 0.006639104, \"time\": 69.5313068151474}, \"5226eb7ff6\": {\"quality\": 0.10946236559139785, \"cost\": 0.008551829, \"time\": 88.08115694522857}, \"526878b5eb\": {\"quality\": 0.05290322580645161, \"cost\": 0.009073641, \"time\": 127.54378099441527}, \"52c1cba6ce\": {\"quality\": 0.11591397849462365, \"cost\": 0.010276206999999999, \"time\": 117.55816102027893}, \"52f041a70e\": {\"quality\": 0.10301075268817204, \"cost\": 0.001223052, \"time\": 42.505560874938965}, \"533867574b\": {\"quality\": 0.12301075268817205, \"cost\": 0.031638318000000006, \"time\": 88.52275938987732}, \"53869388bb\": {\"quality\": 0.05655913978494624, \"cost\": 0.0018472099999999997, \"time\": 38.346792578697205}, \"53aefd41e4\": {\"quality\": 0.10301075268817204, \"cost\": 0.022357914, \"time\": 79.89788761138917}, \"53d2932c4f\": {\"quality\": 0.12374807987711213, \"cost\": 0.007490942, \"time\": 79.30699775218963}, \"54375d3eba\": {\"quality\": 0.04967741935483871, \"cost\": 0.013380047999999999, \"time\": 77.51320762634276}, \"5474247f91\": {\"quality\": 0.0, \"cost\": 0.008527589999999998, \"time\": 80.76838719844818}, \"54993bc472\": {\"quality\": 0.13134408602150538, \"cost\": 0.027307650000000003, \"time\": 45.53245575428009}, \"55358f2285\": {\"quality\": 0.11591397849462365, \"cost\": 0.034442846, \"time\": 74.98782951831818}, \"5569b4f878\": {\"quality\": 0.10623655913978494, \"cost\": 0.004829285, \"time\": 52.80584018230438}, \"557d2cf7ba\": {\"quality\": 0.01935483870967742, \"cost\": 0.009244542000000001, \"time\": 81.70965638160706}, \"55c8aa8935\": {\"quality\": 0.07660522273425499, \"cost\": 0.012967038, \"time\": 127.87031970024108}, \"55e6bf8f14\": {\"quality\": 0.10623655913978494, \"cost\": 0.031587628000000006, \"time\": 88.87307133674622}, \"56a0660622\": {\"quality\": 0.07333333333333333, \"cost\": 0.02785145, \"time\": 61.1418399810791}, \"56a29a28c5\": {\"quality\": 0.04, \"cost\": 0.005067498, \"time\": 67.83964855670928}, \"56c4fd5056\": {\"quality\": 0.0, \"cost\": 0.05983196800000001, \"time\": 88.72949783802034}, \"5703697dbd\": {\"quality\": 0.09268817204301075, \"cost\": 0.006353454, \"time\": 94.9607982635498}, \"5718f2ed80\": {\"quality\": 0.10946236559139785, \"cost\": 0.003271383, \"time\": 89.07395238876343}, \"572a02a59a\": {\"quality\": 0.15623655913978496, \"cost\": 0.004805790000000001, \"time\": 95.1497330904007}, \"5750713a41\": {\"quality\": 0.10301075268817204, \"cost\": 0.006427176, \"time\": 93.20527319908142}, \"57757ef15e\": {\"quality\": 0.06301075268817204, \"cost\": 0.006945941999999999, \"time\": 92.32843782901764}, \"579c81bbe0\": {\"quality\": 0.0, \"cost\": 0.005599266, \"time\": 66.2362874507904}, \"57bed1722f\": {\"quality\": 0.0, \"cost\": 0.005544702, \"time\": 79.84823746681214}, \"585ba6d20b\": {\"quality\": 0.10301075268817204, \"cost\": 0.007482849, \"time\": 135.71461656093598}, \"589267ac64\": {\"quality\": 0.043010752688172046, \"cost\": 0.0073127800000000005, \"time\": 65.76824603080749}, \"589a1cea79\": {\"quality\": 0.03612903225806452, \"cost\": 0.009729424, \"time\": 144.28782908916475}, \"58ca42839b\": {\"quality\": 0.10301075268817204, \"cost\": 0.026166294, \"time\": 107.9141107559204}, \"58dc373441\": {\"quality\": 0.10623655913978494, \"cost\": 0.031231837999999998, \"time\": 134.37797992229463}, \"59006532b4\": {\"quality\": 0.10301075268817204, \"cost\": 0.007373505, \"time\": 106.22529966831206}, \"59326c4e00\": {\"quality\": 0.0, \"cost\": 0.006372392, \"time\": 76.82031381130219}, \"593975c75b\": {\"quality\": 0.11268817204301075, \"cost\": 0.04324769700000001, \"time\": 137.53491501808168}, \"596b4f8694\": {\"quality\": 0.10301075268817204, \"cost\": 0.034689392, \"time\": 136.43740742206575}, \"5971ba4e0d\": {\"quality\": 0.10946236559139785, \"cost\": 0.00188677, \"time\": 45.389907240867615}, \"5996465c0a\": {\"quality\": 0.10623655913978494, \"cost\": 0.031329037000000004, \"time\": 141.25967452526095}, \"59d70b9f65\": {\"quality\": 0.10301075268817204, \"cost\": 0.03321219, \"time\": 110.12462825775147}, \"59f887b67c\": {\"quality\": 0.04, \"cost\": 0.010642958000000001, \"time\": 167.4949809551239}, \"5a22920db4\": {\"quality\": 0.10623655913978494, \"cost\": 0.0019284089999999999, \"time\": 76.70821943283082}, \"5a35020d45\": {\"quality\": 0.10623655913978494, \"cost\": 0.006820569, \"time\": 120.13344073295593}, \"5aa43da1fc\": {\"quality\": 0.11591397849462365, \"cost\": 0.028597824, \"time\": 106.1217529296875}, \"5ae0d88127\": {\"quality\": 0.0, \"cost\": 0.032261024000000006, \"time\": 121.66821670532227}, \"5b10fbdbe1\": {\"quality\": 0.11913978494623656, \"cost\": 0.014016424, \"time\": 136.450377035141}, \"5bade9eb85\": {\"quality\": 0.10623655913978494, \"cost\": 0.009162800999999998, \"time\": 132.39995594024657}, \"5be16744bf\": {\"quality\": 0.09729646697388633, \"cost\": 0.01510615, \"time\": 108.84415483474731}, \"5c5055e252\": {\"quality\": 0.0696774193548387, \"cost\": 0.029808262000000006, \"time\": 102.84497191905976}, \"5c53feccd9\": {\"quality\": 0.09333333333333334, \"cost\": 0.016134796, \"time\": 133.5285586118698}, \"5c77c7c2b2\": {\"quality\": 0.08301075268817204, \"cost\": 0.009062296000000001, \"time\": 106.83322401046752}, \"5d298b5b48\": {\"quality\": 0.10623655913978494, \"cost\": 0.037192654000000006, \"time\": 126.51887106895447}, \"5d41515d2e\": {\"quality\": 0.10946236559139785, \"cost\": 0.027884363000000002, \"time\": 88.88220546245574}, \"5d4babc723\": {\"quality\": 0.12623655913978493, \"cost\": 0.033099258000000006, \"time\": 115.31343786716462}, \"5d79b50feb\": {\"quality\": 0.02258064516129032, \"cost\": 0.013859904000000003, \"time\": 97.84079582691191}, \"5dc216cd6b\": {\"quality\": 0.10946236559139785, \"cost\": 0.010709733999999999, \"time\": 78.82858054637909}, \"5dd68c1b8f\": {\"quality\": 0.10301075268817204, \"cost\": 0.0015060479999999998, \"time\": 49.43739776611328}, \"5de4a882c1\": {\"quality\": 0.10623655913978494, \"cost\": 0.008982037, \"time\": 78.31139621734619}, \"5e04e1c72d\": {\"quality\": 0.10623655913978494, \"cost\": 0.040715788, \"time\": 81.60498032569885}, \"5e923cee9e\": {\"quality\": 0.10301075268817204, \"cost\": 0.017039563, \"time\": 113.84931592941284}, \"5ea2fab380\": {\"quality\": 0.08301075268817204, \"cost\": 0.004995846, \"time\": 52.83791854381562}, \"5eb3bb525b\": {\"quality\": 0.05333333333333333, \"cost\": 0.009461896000000001, \"time\": 96.70504157543182}, \"5eea899380\": {\"quality\": 0.11591397849462365, \"cost\": 0.037967743000000005, \"time\": 113.0098082780838}, \"5f37b3902b\": {\"quality\": 0.06258064516129032, \"cost\": 0.007317896000000001, \"time\": 72.8305846452713}, \"5f9282df3c\": {\"quality\": 0.13946236559139785, \"cost\": 0.018534208000000003, \"time\": 113.41562910079956}, \"6019884cf3\": {\"quality\": 0.10301075268817204, \"cost\": 0.035076015, \"time\": 124.4206553697586}, \"606352363e\": {\"quality\": 0.06301075268817204, \"cost\": 0.036100772, \"time\": 120.98173689842224}, \"608728f868\": {\"quality\": 0.08946236559139785, \"cost\": 0.033319759000000004, \"time\": 122.43799624443054}, \"60b9e936f1\": {\"quality\": 0.02, \"cost\": 0.027687884000000003, \"time\": 62.59779427051544}, \"60cb623c53\": {\"quality\": 0.0, \"cost\": 0.002459736, \"time\": 58.33065054416656}, \"6234de86b4\": {\"quality\": 0.0696774193548387, \"cost\": 0.04136126700000001, \"time\": 96.66275715827942}, \"62352c6854\": {\"quality\": 0.11591397849462365, \"cost\": 0.03903031800000001, \"time\": 117.9777235031128}, \"63a0aaebed\": {\"quality\": 0.03333333333333333, \"cost\": 0.002063613, \"time\": 61.364303207397455}, \"647dda686f\": {\"quality\": 0.11268817204301075, \"cost\": 0.016785803000000002, \"time\": 112.19475474357606}, \"6511b21ded\": {\"quality\": 0.10301075268817204, \"cost\": 0.013076242000000002, \"time\": 83.33274998664857}, \"652c0f4bdf\": {\"quality\": 0.02, \"cost\": 0.013811997999999999, \"time\": 122.77105476856232}, \"6533c85913\": {\"quality\": 0.056129032258064517, \"cost\": 0.010490979000000001, \"time\": 115.75537250041961}, \"65627426e0\": {\"quality\": 0.07333333333333333, \"cost\": 0.000720418, \"time\": 74.83012104034424}, \"65b76da9c6\": {\"quality\": 0.0696774193548387, \"cost\": 0.007407492, \"time\": 86.19350485801698}, \"65be1c1306\": {\"quality\": 0.08623655913978495, \"cost\": 0.001827702, \"time\": 35.30676467418671}, \"65e0216208\": {\"quality\": 0.0, \"cost\": 0.008113563, \"time\": 120.1523479938507}, \"65eee615d7\": {\"quality\": 0.12301075268817205, \"cost\": 0.00056168, \"time\": 33.19386031627655}, \"6623d7a5ac\": {\"quality\": 0.11591397849462365, \"cost\": 0.035060653000000004, \"time\": 117.17992658615111}, \"66750c0934\": {\"quality\": 0.10301075268817204, \"cost\": 0.007793503, \"time\": 92.1173137664795}, \"66776ec181\": {\"quality\": 0.09612903225806452, \"cost\": 0.007265367, \"time\": 135.96958916187288}, \"66e5ae0a21\": {\"quality\": 0.0, \"cost\": 0.021849754, \"time\": 69.76327166557311}, \"6750a8d7a7\": {\"quality\": 0.11591397849462365, \"cost\": 0.014042485, \"time\": 90.68010911941528}, \"67632141f6\": {\"quality\": 0.008333333333333333, \"cost\": 0.0051496020000000005, \"time\": 66.89973094463349}, \"67868fcff6\": {\"quality\": 0.11591397849462365, \"cost\": 0.01194172, \"time\": 93.14265236854553}, \"67bab6732d\": {\"quality\": 0.07612903225806453, \"cost\": 0.011601776, \"time\": 102.1346355676651}, \"67fe399cf1\": {\"quality\": 0.11913978494623656, \"cost\": 0.007208916, \"time\": 103.40100402832032}, \"6846bd8fb3\": {\"quality\": 0.09333333333333334, \"cost\": 0.007100624999999999, \"time\": 104.9791820526123}, \"68b4cc3e39\": {\"quality\": 0.12301075268817205, \"cost\": 0.045394822, \"time\": 143.03540830612184}, \"69a029ae36\": {\"quality\": 0.0, \"cost\": 0.028152226000000002, \"time\": 119.96623668670654}, \"69b3b67de6\": {\"quality\": 0.11591397849462365, \"cost\": 0.008464616999999999, \"time\": 114.51782112121583}, \"69bf3f6ba0\": {\"quality\": 0.08946236559139785, \"cost\": 0.009020988, \"time\": 113.73228361606598}, \"6a022c3f73\": {\"quality\": 0.06946236559139785, \"cost\": 0.004192794, \"time\": 110.7649359703064}, \"6a10c53ad8\": {\"quality\": 0.12697388632872503, \"cost\": 0.012509873000000001, \"time\": 133.7932172060013}, \"6a6348f69d\": {\"quality\": 0.04, \"cost\": 0.001041536, \"time\": 82.5766867876053}, \"6a74a11bee\": {\"quality\": 0.12623655913978493, \"cost\": 0.030354143, \"time\": 111.96903376579286}, \"6a90ec29dd\": {\"quality\": 0.11591397849462365, \"cost\": 0.01421163, \"time\": 101.9924022436142}, \"6aac59742a\": {\"quality\": 0.00967741935483871, \"cost\": 0.009429429, \"time\": 145.55764169692992}, \"6b0862c597\": {\"quality\": 0.10623655913978494, \"cost\": 0.028999667000000007, \"time\": 90.02094576358795}, \"6b3c16def2\": {\"quality\": 0.10697388632872504, \"cost\": 0.001518831, \"time\": 61.66853258609771}, \"6b99b0e901\": {\"quality\": 0.20301075268817204, \"cost\": 0.01664123, \"time\": 88.09062700271608}, \"6b9b2b3515\": {\"quality\": 0.07333333333333333, \"cost\": 0.039732264, \"time\": 98.20316002368926}, \"6bcc02962b\": {\"quality\": 0.10946236559139785, \"cost\": 0.031133424, \"time\": 97.9912957429886}, \"6c1987a9e3\": {\"quality\": 0.12052227342549923, \"cost\": 0.034681694000000006, \"time\": 130.62333233356475}, \"6c50123ee1\": {\"quality\": 0.10301075268817204, \"cost\": 0.027581610000000003, \"time\": 51.59311645030975}, \"6c67c36480\": {\"quality\": 0.02, \"cost\": 0.034065069, \"time\": 137.40613181591033}, \"6c9b9f1363\": {\"quality\": 0.10623655913978494, \"cost\": 0.033907465, \"time\": 132.98066565990447}, \"6cc813aa68\": {\"quality\": 0.04258064516129032, \"cost\": 0.006083242000000001, \"time\": 64.82899880409241}, \"6d20c6ace0\": {\"quality\": 0.12301075268817205, \"cost\": 0.026338517999999998, \"time\": 129.11275482177734}, \"6d444fe21a\": {\"quality\": 0.08623655913978495, \"cost\": 0.011278744, \"time\": 98.86997501850128}, \"6db70dc3b6\": {\"quality\": 0.07333333333333333, \"cost\": 0.001547284, \"time\": 91.66993854045867}, \"6e0690f576\": {\"quality\": 0.02, \"cost\": 0.001789902, \"time\": 73.95127286911011}, \"6e06cc804f\": {\"quality\": 0.10301075268817204, \"cost\": 0.06912567600000001, \"time\": 111.87644176483155}, \"6e24048a2e\": {\"quality\": 0.10623655913978494, \"cost\": 0.020128832, \"time\": 86.35281929969787}, \"6e93514f45\": {\"quality\": 0.0, \"cost\": 0.005360688000000001, \"time\": 106.13040091991425}, \"6eae47102b\": {\"quality\": 0.10301075268817204, \"cost\": 0.008400768, \"time\": 124.15161423683166}, \"6ed4cae469\": {\"quality\": 0.10301075268817204, \"cost\": 0.007222452, \"time\": 52.79307363033295}, \"6ef3b7127e\": {\"quality\": 0.20946236559139786, \"cost\": 0.006944608000000001, \"time\": 77.88400571346283}, \"6f60a05c33\": {\"quality\": 0.06623655913978495, \"cost\": 0.0043084709999999995, \"time\": 119.35040576457976}, \"6fe0b3f929\": {\"quality\": 0.0, \"cost\": 0.006453737999999999, \"time\": 100.1590167760849}, \"700ab1d309\": {\"quality\": 0.0, \"cost\": 0.017191588, \"time\": 130.72283561229705}, \"7040e83d52\": {\"quality\": 0.07333333333333333, \"cost\": 0.021663679, \"time\": 123.93634688854218}, \"7046765af8\": {\"quality\": 0.10301075268817204, \"cost\": 0.035322934, \"time\": 155.01265740394592}, \"70b7c92ce8\": {\"quality\": 0.11456989247311827, \"cost\": 0.030668973000000002, \"time\": 145.30480256080628}, \"70c850e039\": {\"quality\": 0.0696774193548387, \"cost\": 0.0025364249999999997, \"time\": 93.23688249588012}, \"7112a7e64c\": {\"quality\": 0.10946236559139785, \"cost\": 0.005931566000000001, \"time\": 86.47371196746826}, \"71b615468b\": {\"quality\": 0.06, \"cost\": 0.011086878000000001, \"time\": 121.78951773643495}, \"723fd5589a\": {\"quality\": 0.0, \"cost\": 0.013953277000000002, \"time\": 128.76966230869294}, \"7250da0f41\": {\"quality\": 0.0, \"cost\": 0.007366451999999999, \"time\": 91.1376808166504}, \"7274a50778\": {\"quality\": 0.10301075268817204, \"cost\": 0.033229607999999994, \"time\": 102.63909165859224}, \"72d022ce33\": {\"quality\": 0.10301075268817204, \"cost\": 0.04128840500000001, \"time\": 126.40671186447145}, \"7347cf0308\": {\"quality\": 0.0, \"cost\": 0.0036175589999999994, \"time\": 66.07082681655885}, \"736e652158\": {\"quality\": 0.0, \"cost\": 0.006011901, \"time\": 124.95026433467865}, \"739b1f81dc\": {\"quality\": 0.10301075268817204, \"cost\": 0.001929351, \"time\": 59.82662603855133}, \"742a5c0552\": {\"quality\": 0.10623655913978494, \"cost\": 0.028237606000000005, \"time\": 58.23579893112183}, \"742ec1b2e1\": {\"quality\": 0.049677419354838714, \"cost\": 0.029807655000000002, \"time\": 101.90871527194977}, \"7435fd54f8\": {\"quality\": 0.06946236559139785, \"cost\": 0.030427277000000003, \"time\": 121.13120923042297}, \"7445d99939\": {\"quality\": 0.00967741935483871, \"cost\": 0.0071708580000000004, \"time\": 93.88335957527161}, \"7466a5f424\": {\"quality\": 0.21591397849462365, \"cost\": 0.015802706, \"time\": 113.56059277057648}, \"74a0be215b\": {\"quality\": 0.10623655913978494, \"cost\": 0.033583158, \"time\": 125.14361772537231}, \"74d7f64b8c\": {\"quality\": 0.0, \"cost\": 0.006800163, \"time\": 146.70624086856844}, \"7524905580\": {\"quality\": 0.10623655913978494, \"cost\": 0.004893289, \"time\": 87.59031102657318}, \"752d9649f2\": {\"quality\": 0.11268817204301075, \"cost\": 0.013868582, \"time\": 121.90489645004273}, \"75d61c2cd0\": {\"quality\": 0.10301075268817204, \"cost\": 0.029877515, \"time\": 85.95328326225281}, \"7604c0aa13\": {\"quality\": 0.10301075268817204, \"cost\": 0.010774968000000001, \"time\": 84.55494666099548}, \"76c09db721\": {\"quality\": 0.10301075268817204, \"cost\": 0.016476853, \"time\": 132.14744505882265}, \"774f268b66\": {\"quality\": 0.10946236559139785, \"cost\": 0.038069169, \"time\": 123.8245237827301}, \"7765576286\": {\"quality\": 0.09333333333333334, \"cost\": 0.008933451, \"time\": 130.5573842048645}, \"77c02b00c1\": {\"quality\": 0.05655913978494624, \"cost\": 0.011193958, \"time\": 124.06667771339417}, \"7801da66b9\": {\"quality\": 0.0, \"cost\": 0.004612473000000001, \"time\": 94.23449354171753}, \"782d52674e\": {\"quality\": 0.10301075268817204, \"cost\": 0.034843558000000004, \"time\": 141.27741813659668}, \"786e5d0af5\": {\"quality\": 0.10946236559139785, \"cost\": 0.009984510999999998, \"time\": 130.25511062145233}, \"7878563d63\": {\"quality\": 0.10946236559139785, \"cost\": 0.009147046, \"time\": 73.60474834442138}, \"79e1ca9b3c\": {\"quality\": 0.0, \"cost\": 0.02973492, \"time\": 82.63413181304932}, \"79fad58f07\": {\"quality\": 0.12623655913978493, \"cost\": 0.002492818, \"time\": 58.73518245220184}, \"7a207b42a8\": {\"quality\": 0.0, \"cost\": 0.008606742, \"time\": 142.3234792470932}, \"7a2cdc546c\": {\"quality\": 0.10301075268817204, \"cost\": 0.032290785, \"time\": 102.24121663570403}, \"7a42a77788\": {\"quality\": 0.10047619047619048, \"cost\": 0.031140232, \"time\": 94.29256238937378}, \"7a58d3472b\": {\"quality\": 0.09333333333333334, \"cost\": 0.013635729000000001, \"time\": 102.35271875858307}, \"7b3937c1f1\": {\"quality\": 0.10301075268817204, \"cost\": 0.009220510000000001, \"time\": 54.74555864334106}, \"7b6f44618e\": {\"quality\": 0.06946236559139785, \"cost\": 0.013006586, \"time\": 130.8397382736206}, \"7c62576527\": {\"quality\": 0.22424731182795699, \"cost\": 0.023333732000000003, \"time\": 129.530002951622}, \"7c89a2b69e\": {\"quality\": 0.0, \"cost\": 0.012572941, \"time\": 131.1510176181793}, \"7c96c9712f\": {\"quality\": 0.12301075268817205, \"cost\": 0.02800367, \"time\": 71.56067564487458}, \"7ca066aa1c\": {\"quality\": 0.10301075268817204, \"cost\": 0.017135469, \"time\": 126.58104584217071}, \"7cb5591f27\": {\"quality\": 0.12301075268817205, \"cost\": 0.037874200000000004, \"time\": 99.9374593257904}, \"7cf56a7fbc\": {\"quality\": 0.0, \"cost\": 0.03754542600000001, \"time\": 81.54608149528502}, \"7d44f0959d\": {\"quality\": 0.11015360983102919, \"cost\": 0.006577314, \"time\": 95.78297588825225}, \"7d60c38c5c\": {\"quality\": 0.10301075268817204, \"cost\": 0.0019278, \"time\": 62.340004205703735}, \"7d9b4535ac\": {\"quality\": 0.12301075268817205, \"cost\": 0.030719909000000004, \"time\": 124.86560246944427}, \"7daf7ff182\": {\"quality\": 0.21591397849462365, \"cost\": 0.011567735999999999, \"time\": 86.85500297546386}, \"7dcedb3d02\": {\"quality\": 0.11591397849462365, \"cost\": 0.034740937, \"time\": 118.09863307476044}, \"7e22f12cd1\": {\"quality\": 0.10946236559139785, \"cost\": 0.012891924999999999, \"time\": 121.22575757503509}, \"7e53a50b13\": {\"quality\": 0.10946236559139785, \"cost\": 0.014215881, \"time\": 90.20128819942474}, \"7ed07ad40a\": {\"quality\": 0.10623655913978494, \"cost\": 0.006843809999999999, \"time\": 95.13763875961303}, \"7fa67a7656\": {\"quality\": 0.06623655913978495, \"cost\": 0.008441146000000002, \"time\": 80.71841621398926}, \"7fc6c84bdf\": {\"quality\": 0.10301075268817204, \"cost\": 0.014648633, \"time\": 94.4585831642151}, \"7ff8a779cc\": {\"quality\": 0.049677419354838714, \"cost\": 0.033345343, \"time\": 90.90201032161713}, \"801af99400\": {\"quality\": 0.12623655913978493, \"cost\": 0.031287835, \"time\": 92.26307544708251}, \"806881adcb\": {\"quality\": 0.10623655913978494, \"cost\": 0.009180413, \"time\": 103.7968935251236}, \"80ad4122e4\": {\"quality\": 0.10946236559139785, \"cost\": 0.033705702000000004, \"time\": 147.08289949893953}, \"80be7df955\": {\"quality\": 0.0, \"cost\": 0.001751958, \"time\": 72.68495740890503}, \"80bf60c422\": {\"quality\": 0.0, \"cost\": 0.002103153, \"time\": 69.09833533763886}, \"81333c7a33\": {\"quality\": 0.19333333333333333, \"cost\": 0.018656088, \"time\": 122.73940353393554}, \"813e75210b\": {\"quality\": 0.12623655913978493, \"cost\": 0.002453274, \"time\": 31.607514262199402}, \"81660ae8b2\": {\"quality\": 0.00967741935483871, \"cost\": 0.03510192400000001, \"time\": 159.472354388237}, \"816958b5d1\": {\"quality\": 0.10301075268817204, \"cost\": 0.033492955000000005, \"time\": 150.83300273418428}, \"81ab2ef3f4\": {\"quality\": 0.08301075268817204, \"cost\": 0.015009652000000002, \"time\": 122.83408730030061}, \"829df73946\": {\"quality\": 0.10623655913978494, \"cost\": 0.0027339649999999997, \"time\": 95.04270524978638}, \"82ea1bd1b9\": {\"quality\": 0.10301075268817204, \"cost\": 0.036581163, \"time\": 130.3844569683075}, \"8357183895\": {\"quality\": 0.0, \"cost\": 0.009140319000000001, \"time\": 168.7040484428406}, \"8392a6083a\": {\"quality\": 0.11591397849462365, \"cost\": 0.013724715999999998, \"time\": 170.04993345737458}, \"83aee532b7\": {\"quality\": 0.0, \"cost\": 0.032044618999999996, \"time\": 169.97980024814606}, \"83b26646c3\": {\"quality\": 0.10946236559139785, \"cost\": 0.027026598000000002, \"time\": 126.73678522109986}, \"83c9e66ec6\": {\"quality\": 0.10623655913978494, \"cost\": 0.0077078630000000006, \"time\": 122.85937674045563}, \"847a0e5db5\": {\"quality\": 0.10301075268817204, \"cost\": 0.033220362, \"time\": 118.9956719636917}, \"847fd49235\": {\"quality\": 0.10623655913978494, \"cost\": 0.004587912, \"time\": 120.00769526958466}, \"849100224d\": {\"quality\": 0.11729646697388633, \"cost\": 0.040131096000000005, \"time\": 162.80983967781066}, \"84b91c37ab\": {\"quality\": 0.10301075268817204, \"cost\": 0.035888904, \"time\": 185.50609340667725}, \"8519bef585\": {\"quality\": 0.0, \"cost\": 0.002883114, \"time\": 83.83753573894501}, \"85c94a5505\": {\"quality\": 0.10623655913978494, \"cost\": 0.003060162, \"time\": 47.44887166023254}, \"85eda38404\": {\"quality\": 0.06301075268817204, \"cost\": 0.00743115, \"time\": 97.06213176250458}, \"862183bfb9\": {\"quality\": 0.11591397849462365, \"cost\": 0.007656215, \"time\": 150.60711097717285}, \"8631e49c94\": {\"quality\": 0.10623655913978494, \"cost\": 0.032397725, \"time\": 178.85168232917786}, \"8668f65f05\": {\"quality\": 0.10301075268817204, \"cost\": 0.010037819, \"time\": 170.7291125535965}, \"86bf6375af\": {\"quality\": 0.07290322580645162, \"cost\": 0.034898615, \"time\": 167.93282201290128}, \"870e2f87b4\": {\"quality\": 0.09333333333333334, \"cost\": 0.014351701000000001, \"time\": 174.38446729183198}, \"887ad124e1\": {\"quality\": 0.10623655913978494, \"cost\": 0.0033872489999999997, \"time\": 141.54452052116392}, \"8886cb3082\": {\"quality\": 0.10623655913978494, \"cost\": 0.010027301, \"time\": 168.62686750888824}, \"88e71efa9b\": {\"quality\": 0.11591397849462365, \"cost\": 0.024657334, \"time\": 163.67923073768617}, \"8941621423\": {\"quality\": 0.10623655913978494, \"cost\": 0.006795555, \"time\": 135.67179551124573}, \"8950a6efe0\": {\"quality\": 0.10946236559139785, \"cost\": 0.029011140000000005, \"time\": 120.09507937431336}, \"8961e4d901\": {\"quality\": 0.02, \"cost\": 0.007544093999999999, \"time\": 115.46372551918029}, \"8974aa89a0\": {\"quality\": 0.08301075268817204, \"cost\": 0.0016073620000000002, \"time\": 48.72573773860931}, \"89836d2020\": {\"quality\": 0.21591397849462365, \"cost\": 0.03316775000000001, \"time\": 93.58950910568237}, \"89a289907e\": {\"quality\": 0.10301075268817204, \"cost\": 0.009082531000000001, \"time\": 127.31396248340606}, \"89a35a09b1\": {\"quality\": 0.10623655913978494, \"cost\": 0.007982379000000001, \"time\": 124.18780100345612}, \"8a3a35c762\": {\"quality\": 0.10946236559139785, \"cost\": 0.03612732600000001, \"time\": 119.14178504943848}, \"8a50695d1f\": {\"quality\": 0.043010752688172046, \"cost\": 0.016531714000000003, \"time\": 100.70311589241028}, \"8ab351aa13\": {\"quality\": 0.0, \"cost\": 0.040063444000000004, \"time\": 105.603106880188}, \"8ac8b5773a\": {\"quality\": 0.0696774193548387, \"cost\": 0.033945373, \"time\": 124.25757415294648}, \"8acd758b7f\": {\"quality\": 0.10623655913978494, \"cost\": 0.004647693, \"time\": 101.40544395446777}, \"8b10891ea5\": {\"quality\": 0.11333333333333334, \"cost\": 0.038892921999999996, \"time\": 124.35444567203521}, \"8b721bbc6f\": {\"quality\": 0.10946236559139785, \"cost\": 0.008056791, \"time\": 122.71747550964355}, \"8b77535cce\": {\"quality\": 0.12623655913978493, \"cost\": 0.035407286999999996, \"time\": 130.12753999233246}, \"8bbbe0f52a\": {\"quality\": 0.09279569892473119, \"cost\": 0.006340248, \"time\": 96.97207028865814}, \"8bc184f385\": {\"quality\": 0.10623655913978494, \"cost\": 0.0073677989999999995, \"time\": 97.76727423667907}, \"8bf5c3eadc\": {\"quality\": 0.02, \"cost\": 0.006312036, \"time\": 82.65424501895905}, \"8bf80a50cb\": {\"quality\": 0.11591397849462365, \"cost\": 0.0346818, \"time\": 110.19822652339934}, \"8c274ca255\": {\"quality\": 0.12301075268817205, \"cost\": 0.036505015, \"time\": 115.35032222270965}, \"8d7594020b\": {\"quality\": 0.12623655913978493, \"cost\": 0.029785446, \"time\": 79.58442108631134}, \"8d79e03266\": {\"quality\": 0.10301075268817204, \"cost\": 0.008962553, \"time\": 91.70296292304992}, \"8d90814b94\": {\"quality\": 0.21591397849462365, \"cost\": 0.009549217000000002, \"time\": 107.88517692089081}, \"8e1a01da19\": {\"quality\": 0.00967741935483871, \"cost\": 0.030469137000000007, \"time\": 110.28689606189728}, \"8e2498635d\": {\"quality\": 0.12946236559139784, \"cost\": 0.067304136, \"time\": 88.78618555068971}, \"8e5842ccbd\": {\"quality\": 0.12623655913978493, \"cost\": 0.0035327910000000004, \"time\": 61.36363306045533}, \"8e5daf241e\": {\"quality\": 0.10301075268817204, \"cost\": 0.029329670000000002, \"time\": 80.64679687023164}, \"8e9715ee01\": {\"quality\": 0.13268817204301075, \"cost\": 0.034420532000000004, \"time\": 73.95220372676849}, \"8e9b7300d4\": {\"quality\": 0.10301075268817204, \"cost\": 0.029812478000000003, \"time\": 81.88405811786652}, \"8f29fab8ac\": {\"quality\": 0.05333333333333333, \"cost\": 0.034482348, \"time\": 119.19576361179352}, \"8f44d89429\": {\"quality\": 0.20301075268817204, \"cost\": 0.034302998, \"time\": 86.89617729187012}, \"8f4caddfe6\": {\"quality\": 0.07935483870967741, \"cost\": 0.014937176, \"time\": 100.91416237354278}, \"8f4edde3f0\": {\"quality\": 0.10623655913978494, \"cost\": 0.017718742000000003, \"time\": 124.73474152088164}, \"8f9cefbc22\": {\"quality\": 0.10623655913978494, \"cost\": 0.028400606000000002, \"time\": 59.505778431892395}, \"9025e2480f\": {\"quality\": 0.11268817204301075, \"cost\": 0.006730606, \"time\": 74.51114990711213}, \"9028588af4\": {\"quality\": 0.10623655913978494, \"cost\": 0.00524569, \"time\": 45.83101971149445}, \"9059fd80ad\": {\"quality\": 0.00967741935483871, \"cost\": 0.0016962140000000001, \"time\": 79.82600400447845}, \"90d5e40c1b\": {\"quality\": 0.0, \"cost\": 0.006609534, \"time\": 97.84093914031982}, \"90d9a86a2a\": {\"quality\": 0.0, \"cost\": 0.0016498349999999997, \"time\": 50.78291637897492}, \"90ff13783c\": {\"quality\": 0.10301075268817204, \"cost\": 0.010615371000000002, \"time\": 137.4936174631119}, \"90ff8eb055\": {\"quality\": 0.029677419354838707, \"cost\": 0.010129776, \"time\": 135.72056851387023}, \"9104e31369\": {\"quality\": 0.09333333333333334, \"cost\": 0.013592545000000001, \"time\": 145.33199377059935}, \"918983323f\": {\"quality\": 0.00967741935483871, \"cost\": 0.0012531780000000002, \"time\": 41.7289758682251}, \"91928dfdd9\": {\"quality\": 0.10623655913978494, \"cost\": 0.029831955000000007, \"time\": 108.31667373180389}, \"91c800af6b\": {\"quality\": 0.08301075268817204, \"cost\": 0.011924733, \"time\": 149.60272026062012}, \"91e841cfd5\": {\"quality\": 0.11268817204301075, \"cost\": 0.035107227000000005, \"time\": 144.8135196208954}, \"9253901a1f\": {\"quality\": 0.0, \"cost\": 0.038044404000000004, \"time\": 144.24513857364656}, \"9288642e53\": {\"quality\": 0.02, \"cost\": 0.006087138000000001, \"time\": 72.39175655841828}, \"92ba9c5be3\": {\"quality\": 0.13591397849462367, \"cost\": 0.030314066, \"time\": 89.64683423042297}, \"92c9dcd43b\": {\"quality\": 0.10301075268817204, \"cost\": 0.009152825, \"time\": 133.89202611446382}, \"93011c0821\": {\"quality\": 0.03935483870967742, \"cost\": 0.012697265000000001, \"time\": 135.9506582260132}, \"933b4d17dd\": {\"quality\": 0.10301075268817204, \"cost\": 0.004906763, \"time\": 94.6856369972229}, \"9373267bdb\": {\"quality\": 0.12623655913978493, \"cost\": 0.03234980700000001, \"time\": 123.84810786247253}, \"94010928c6\": {\"quality\": 0.08623655913978495, \"cost\": 0.007078176, \"time\": 105.30463340282441}, \"9403809e44\": {\"quality\": 0.08301075268817204, \"cost\": 0.012096366, \"time\": 135.0477834701538}, \"940c88ddc5\": {\"quality\": 0.06946236559139785, \"cost\": 0.010277144, \"time\": 97.58399150371551}, \"948f4081ba\": {\"quality\": 0.10623655913978494, \"cost\": 0.015952197, \"time\": 132.39694006443023}, \"94ac356663\": {\"quality\": 0.10301075268817204, \"cost\": 0.007538082, \"time\": 100.63510990142822}, \"94dff9a424\": {\"quality\": 0.10301075268817204, \"cost\": 0.0058817290000000005, \"time\": 97.47921371459961}, \"9508356a2e\": {\"quality\": 0.0, \"cost\": 0.0074193060000000005, \"time\": 119.1678344964981}, \"9539d0e28c\": {\"quality\": 0.11913978494623656, \"cost\": 0.016965942000000005, \"time\": 127.95936200618743}, \"956bdcc254\": {\"quality\": 0.10301075268817204, \"cost\": 0.016353198000000003, \"time\": 113.47469205856324}, \"957d0dafc5\": {\"quality\": 0.12623655913978493, \"cost\": 0.03451419800000001, \"time\": 99.19511275291444}, \"9594b0c783\": {\"quality\": 0.06946236559139785, \"cost\": 0.0033617670000000003, \"time\": 103.66688406467438}, \"95a7b80c2a\": {\"quality\": 0.10301075268817204, \"cost\": 0.012909755, \"time\": 96.70905361175537}, \"964c671f18\": {\"quality\": 0.21591397849462365, \"cost\": 0.012280253000000001, \"time\": 130.05689902305602}, \"9679fe2b69\": {\"quality\": 0.10301075268817204, \"cost\": 0.003347709, \"time\": 92.87826173305511}, \"968fc95038\": {\"quality\": 0.10301075268817204, \"cost\": 0.0036856379999999993, \"time\": 98.10628998279572}, \"96b487c724\": {\"quality\": 0.08946236559139785, \"cost\": 0.00053203, \"time\": 54.895061421394345}, \"96c30205f5\": {\"quality\": 0.10623655913978494, \"cost\": 0.031391626, \"time\": 97.40448112487793}, \"96f87d6483\": {\"quality\": 0.07333333333333333, \"cost\": 0.012635357, \"time\": 153.3388079404831}, \"972c83b002\": {\"quality\": 0.029677419354838707, \"cost\": 0.007475796, \"time\": 118.32212336063385}, \"977a4d6b6b\": {\"quality\": 0.13591397849462367, \"cost\": 0.01375324, \"time\": 138.46970500946043}, \"97bc30bd83\": {\"quality\": 0.07612903225806453, \"cost\": 0.010801506999999998, \"time\": 111.81023037433624}, \"980db5f95f\": {\"quality\": 0.10301075268817204, \"cost\": 0.03036672, \"time\": 113.69273748397828}, \"9836765d41\": {\"quality\": 0.12301075268817205, \"cost\": 0.032121622, \"time\": 104.68945379257201}, \"99569e3937\": {\"quality\": 0.10623655913978494, \"cost\": 0.035286567000000005, \"time\": 141.43686013221742}, \"99ea16a9a6\": {\"quality\": 0.10623655913978494, \"cost\": 0.009821013, \"time\": 108.473588347435}, \"9a5b39370f\": {\"quality\": 0.0, \"cost\": 0.036045704, \"time\": 155.65953776836395}, \"9aa4abfb50\": {\"quality\": 0.0, \"cost\": 0.003652884, \"time\": 89.60838494300842}, \"9ad7a98c31\": {\"quality\": 0.12301075268817205, \"cost\": 0.03251525400000001, \"time\": 111.1855703830719}, \"9b3fb79bcb\": {\"quality\": 0.12301075268817205, \"cost\": 0.010473083, \"time\": 107.05149478912352}, \"9b6d4915f3\": {\"quality\": 0.0, \"cost\": 0.00475386, \"time\": 53.93124535083771}, \"9bae5bafc1\": {\"quality\": 0.09333333333333334, \"cost\": 0.013743354000000001, \"time\": 105.66356384754181}, \"9be8a5f317\": {\"quality\": 0.10623655913978494, \"cost\": 0.038140224, \"time\": 141.94463317394258}, \"9c549db0a7\": {\"quality\": 0.0, \"cost\": 0.011336479, \"time\": 83.88670737743377}, \"9c85f8cfcb\": {\"quality\": 0.0, \"cost\": 0.002950311, \"time\": 71.82748806476593}, \"9c8cc46e6c\": {\"quality\": 0.06623655913978495, \"cost\": 0.0019796609999999997, \"time\": 72.90286383628845}, \"9c97d35a30\": {\"quality\": 0.10301075268817204, \"cost\": 0.005516262, \"time\": 115.19557769298552}, \"9cbe7858a2\": {\"quality\": 0.11333333333333334, \"cost\": 0.034743496000000006, \"time\": 140.37097189426424}, \"9ce2c3fd98\": {\"quality\": 0.0, \"cost\": 0.004397076, \"time\": 118.44574811458588}, \"9d18cd0737\": {\"quality\": 0.06301075268817204, \"cost\": 0.004985955, \"time\": 77.21244201660156}, \"9e06360bc9\": {\"quality\": 0.10946236559139785, \"cost\": 0.008386245, \"time\": 112.56666424274445}, \"9f07a95e69\": {\"quality\": 0.12301075268817205, \"cost\": 0.039345112, \"time\": 142.53303418159484}, \"9fb157be35\": {\"quality\": 0.04, \"cost\": 0.009244232, \"time\": 117.06155586242676}, \"a04ac8e33a\": {\"quality\": 0.0, \"cost\": 0.030044402000000005, \"time\": 83.63084626197815}, \"a04bc6e116\": {\"quality\": 0.04967741935483871, \"cost\": 0.029843946, \"time\": 100.13187789916992}, \"a0b81be5b4\": {\"quality\": 0.10301075268817204, \"cost\": 0.0022555170000000003, \"time\": 77.33556809425355}, \"a0c85d260e\": {\"quality\": 0.10623655913978494, \"cost\": 0.013999365, \"time\": 147.74334559440612}, \"a0dc9f50ac\": {\"quality\": 0.012903225806451613, \"cost\": 0.006003528, \"time\": 77.70861837863922}, \"a18225b7b5\": {\"quality\": 0.10301075268817204, \"cost\": 0.033204910000000004, \"time\": 99.84152095317842}, \"a1d822289e\": {\"quality\": 0.10301075268817204, \"cost\": 0.03703391, \"time\": 105.16314966678618}, \"a25596c056\": {\"quality\": 0.06301075268817204, \"cost\": 0.034485563999999996, \"time\": 138.92013294696807}, \"a2811c7324\": {\"quality\": 0.10623655913978494, \"cost\": 0.009388681000000001, \"time\": 112.0317987203598}, \"a2aa082d14\": {\"quality\": 0.0, \"cost\": 0.014266204, \"time\": 101.78627176284789}, \"a2cd339ad9\": {\"quality\": 0.09333333333333334, \"cost\": 0.010370297, \"time\": 147.42609903812408}, \"a2fd03e6a5\": {\"quality\": 0.07655913978494623, \"cost\": 0.004336301999999999, \"time\": 36.67434537410736}, \"a31e87d7cb\": {\"quality\": 0.06301075268817204, \"cost\": 0.001581552, \"time\": 76.9124368429184}, \"a3e23c327b\": {\"quality\": 0.06946236559139785, \"cost\": 0.03564489000000001, \"time\": 150.25721719264985}, \"a457f6c300\": {\"quality\": 0.0, \"cost\": 0.007949861999999999, \"time\": 132.3079260349274}, \"a47de025c8\": {\"quality\": 0.0, \"cost\": 0.004268559, \"time\": 89.55218677520752}, \"a515a9c8cc\": {\"quality\": 0.05333333333333333, \"cost\": 0.002060598, \"time\": 76.20932059288026}, \"a5949b76ec\": {\"quality\": 0.0, \"cost\": 0.002426688, \"time\": 69.78927192687988}, \"a60dd076b8\": {\"quality\": 0.10623655913978494, \"cost\": 0.002494836, \"time\": 77.4125370502472}, \"a6297a6c56\": {\"quality\": 0.06623655913978495, \"cost\": 0.0024065099999999997, \"time\": 65.99698326587676}, \"a62b7555b9\": {\"quality\": 0.08258064516129032, \"cost\": 0.03560092200000001, \"time\": 131.32008333206176}, \"a63f48e8ca\": {\"quality\": 0.0, \"cost\": 0.029433226000000003, \"time\": 74.86844005584717}, \"a6460dbb7c\": {\"quality\": 0.10623655913978494, \"cost\": 0.002761025, \"time\": 64.81612529754639}, \"a66a4cf4b0\": {\"quality\": 0.11591397849462365, \"cost\": 0.03962680600000001, \"time\": 131.48869993686674}, \"a6e2d69222\": {\"quality\": 0.10301075268817204, \"cost\": 0.029019002, \"time\": 97.08205032348633}, \"a717c4c535\": {\"quality\": 0.05290322580645161, \"cost\": 0.013103212, \"time\": 130.28173251152037}, \"a76afe9960\": {\"quality\": 0.12301075268817205, \"cost\": 0.030138277, \"time\": 103.21392459869384}, \"a7a6353090\": {\"quality\": 0.10623655913978494, \"cost\": 0.031531417000000006, \"time\": 103.30408613681793}, \"a80f6535b1\": {\"quality\": 0.06301075268817204, \"cost\": 0.005650186, \"time\": 66.3761245250702}, \"a86b137d7f\": {\"quality\": 0.06946236559139785, \"cost\": 0.002503022, \"time\": 67.13813652992249}, \"a88eb1493c\": {\"quality\": 0.10301075268817204, \"cost\": 0.0049795349999999985, \"time\": 64.7448471069336}, \"a89c533d6c\": {\"quality\": 0.0, \"cost\": 0.027630250000000002, \"time\": 46.682720041275026}, \"a8d8264600\": {\"quality\": 0.1296774193548387, \"cost\": 0.035944667, \"time\": 153.82529056072235}, \"a8eb36b210\": {\"quality\": 0.10301075268817204, \"cost\": 0.023975060000000003, \"time\": 108.62396328449249}, \"a95b4a6dd0\": {\"quality\": 0.08301075268817204, \"cost\": 0.00891948, \"time\": 111.40819778442383}, \"a9621ea4e6\": {\"quality\": 0.043010752688172046, \"cost\": 0.00758955, \"time\": 111.94071905612945}, \"a9721a0a50\": {\"quality\": 0.11268817204301075, \"cost\": 0.006963254, \"time\": 83.20404796600342}, \"a972b02c61\": {\"quality\": 0.12301075268817205, \"cost\": 0.016982312, \"time\": 36.22950010299682}, \"a9c5c4e311\": {\"quality\": 0.11456989247311827, \"cost\": 0.009789541, \"time\": 112.16675176620484}, \"a9d96670eb\": {\"quality\": 0.12623655913978493, \"cost\": 0.028751066000000002, \"time\": 111.07726712226868}, \"a9e8c974d3\": {\"quality\": 0.0, \"cost\": 0.03628516, \"time\": 154.63389499187468}, \"aa08180e36\": {\"quality\": 0.07591397849462365, \"cost\": 0.010442482000000001, \"time\": 149.7024934768677}, \"aa38702a02\": {\"quality\": 0.06623655913978495, \"cost\": 0.00453825, \"time\": 115.99718639850616}, \"aadbfc418b\": {\"quality\": 0.15301075268817205, \"cost\": 0.0039624659999999996, \"time\": 115.4608172416687}, \"aaeb8b0010\": {\"quality\": 0.03612903225806452, \"cost\": 0.033459904, \"time\": 146.18061661720276}, \"ab288ee7f2\": {\"quality\": 0.043010752688172046, \"cost\": 0.031835803, \"time\": 151.1489573955536}, \"ab43b02cb0\": {\"quality\": 0.0, \"cost\": 0.010940676, \"time\": 85.55018713474274}, \"aba1d612cc\": {\"quality\": 0.10301075268817204, \"cost\": 0.04242697200000001, \"time\": 129.53015527725222}, \"ac208e7a1d\": {\"quality\": 0.20623655913978495, \"cost\": 0.010061258, \"time\": 87.22576036453248}, \"ac2224adbe\": {\"quality\": 0.0, \"cost\": 0.026542246000000005, \"time\": 73.37120480537413}, \"ac828ffe70\": {\"quality\": 0.07655913978494625, \"cost\": 0.000756074, \"time\": 55.469612050056455}, \"ac9fdc1550\": {\"quality\": 0.10301075268817204, \"cost\": 0.013684735, \"time\": 102.56252949237823}, \"aca957ecff\": {\"quality\": 0.11456989247311827, \"cost\": 0.03768476100000001, \"time\": 152.26708533763883}, \"acfe1ed920\": {\"quality\": 0.09456989247311828, \"cost\": 0.005188744, \"time\": 57.178893375396726}, \"ad3efe44c3\": {\"quality\": 0.0, \"cost\": 0.002697534, \"time\": 72.25062220096589}, \"ad41c95a99\": {\"quality\": 0.10623655913978494, \"cost\": 0.025689668000000002, \"time\": 115.99265196323395}, \"ad48432c22\": {\"quality\": 0.0, \"cost\": 0.009775953, \"time\": 191.9520122528076}, \"ad6ebbba8d\": {\"quality\": 0.0, \"cost\": 0.007917252999999999, \"time\": 140.29350352287292}, \"ad90055ef6\": {\"quality\": 0.0, \"cost\": 0.003889926, \"time\": 112.45582339763641}, \"adab1e0fb1\": {\"quality\": 0.10301075268817204, \"cost\": 0.006015302, \"time\": 91.00434277057647}, \"ae655ec593\": {\"quality\": 0.08301075268817204, \"cost\": 0.004026768, \"time\": 141.7740585565567}, \"ae94b172be\": {\"quality\": 0.06301075268817204, \"cost\": 0.007697588, \"time\": 143.52500240802766}, \"aec9dc5873\": {\"quality\": 0.0, \"cost\": 0.027518260000000003, \"time\": 96.50754041671753}, \"af360c323c\": {\"quality\": 0.0, \"cost\": 0.0033090330000000003, \"time\": 149.9352635383606}, \"af90567194\": {\"quality\": 0.10623655913978494, \"cost\": 0.013733104999999999, \"time\": 181.40714287757874}, \"afe77d0f89\": {\"quality\": 0.11591397849462365, \"cost\": 0.03486629, \"time\": 177.79787800312042}, \"b0948c05b6\": {\"quality\": 0.09333333333333334, \"cost\": 0.01202388, \"time\": 110.92746062278746}, \"b0c4a6640b\": {\"quality\": 0.10301075268817204, \"cost\": 0.031655476, \"time\": 155.44975624084475}, \"b12caafd58\": {\"quality\": 0.06301075268817204, \"cost\": 0.03197457, \"time\": 155.3449794769287}, \"b18168b9c1\": {\"quality\": 0.08301075268817204, \"cost\": 0.0077949479999999995, \"time\": 119.46661601066589}, \"b1a7428a01\": {\"quality\": 0.12301075268817205, \"cost\": 0.034865056000000005, \"time\": 145.9925803422928}, \"b1acdebb48\": {\"quality\": 0.0929032258064516, \"cost\": 0.04751790600000001, \"time\": 124.27810626029968}, \"b1b06f4ee7\": {\"quality\": 0.0, \"cost\": 0.05763750000000001, \"time\": 95.25730528831483}, \"b1cf8d33e5\": {\"quality\": 0.08301075268817204, \"cost\": 0.007401675000000001, \"time\": 103.68885869979857}, \"b1e9ab6b1a\": {\"quality\": 0.10946236559139785, \"cost\": 0.038297786, \"time\": 146.62364199161527}, \"b2b057ba41\": {\"quality\": 0.10301075268817204, \"cost\": 0.00373947, \"time\": 102.88082673549653}, \"b2e063499d\": {\"quality\": 0.21591397849462365, \"cost\": 0.009003760999999999, \"time\": 103.52594914436341}, \"b33412410e\": {\"quality\": 0.0, \"cost\": 0.02474838, \"time\": 82.1103196144104}, \"b3369775dc\": {\"quality\": 0.10301075268817204, \"cost\": 0.012564327, \"time\": 147.59253644943237}, \"b3b9205f60\": {\"quality\": 0.07591397849462365, \"cost\": 0.020412146000000003, \"time\": 143.55707364082338}, \"b3c56f0b3c\": {\"quality\": 0.10623655913978494, \"cost\": 0.04188011999999999, \"time\": 104.512482714653}, \"b3f20b706d\": {\"quality\": 0.0, \"cost\": 0.0030317039999999996, \"time\": 98.11458349227905}, \"b4002173ee\": {\"quality\": 0.08301075268817204, \"cost\": 0.0054744, \"time\": 75.72368431091309}, \"b46d382384\": {\"quality\": 0.10301075268817204, \"cost\": 0.05967477, \"time\": 110.93994162082672}, \"b4b2482ef9\": {\"quality\": 0.07591397849462365, \"cost\": 0.033919934, \"time\": 114.34581046104432}, \"b4be043238\": {\"quality\": 0.12301075268817205, \"cost\": 0.04046709600000001, \"time\": 111.69091956615448}, \"b531bd0548\": {\"quality\": 0.10623655913978494, \"cost\": 0.042357856, \"time\": 138.67396595478056}, \"b56c312eda\": {\"quality\": 0.08301075268817204, \"cost\": 0.008939847, \"time\": 141.99621114730834}, \"b5e2b41c1c\": {\"quality\": 0.10623655913978494, \"cost\": 0.004302350999999999, \"time\": 110.59975845813752}, \"b61ce57a90\": {\"quality\": 0.11591397849462365, \"cost\": 0.020101276, \"time\": 140.674871301651}, \"b64ddb14f9\": {\"quality\": 0.10946236559139785, \"cost\": 0.0024013059999999998, \"time\": 41.2197571516037}, \"b67107a43e\": {\"quality\": 0.05333333333333333, \"cost\": 0.0056747790000000005, \"time\": 151.14496650695799}, \"b67720aa5c\": {\"quality\": 0.0, \"cost\": 0.029933376000000005, \"time\": 112.10192131996155}, \"b682a23b89\": {\"quality\": 0.10301075268817204, \"cost\": 0.004969453, \"time\": 109.97392725944519}, \"b69ef5add4\": {\"quality\": 0.04, \"cost\": 0.033153256000000006, \"time\": 127.61734595298768}, \"b796b7ffd3\": {\"quality\": 0.10946236559139785, \"cost\": 0.009056132000000001, \"time\": 156.37638986110687}, \"b7a0083dc4\": {\"quality\": 0.06, \"cost\": 0.029241969, \"time\": 111.82368907928466}, \"b7d0e8557f\": {\"quality\": 0.06623655913978495, \"cost\": 0.008024637, \"time\": 146.9499261379242}, \"b8317a3a8c\": {\"quality\": 0.10301075268817204, \"cost\": 0.040210949, \"time\": 126.69586458206176}, \"b8ab3d2f25\": {\"quality\": 0.11591397849462365, \"cost\": 0.008929705999999999, \"time\": 119.44116389751434}, \"b8b569172f\": {\"quality\": 0.10301075268817204, \"cost\": 0.010143565999999998, \"time\": 119.48254477977753}, \"b8f5ab44bb\": {\"quality\": 0.13591397849462367, \"cost\": 0.039590358000000006, \"time\": 144.63666880130768}, \"b91e7fdb29\": {\"quality\": 0.0, \"cost\": 0.009813995999999998, \"time\": 178.9033116340637}, \"b932beaaa6\": {\"quality\": 0.10301075268817204, \"cost\": 0.03316909800000001, \"time\": 101.8533546447754}, \"b9770c2261\": {\"quality\": 0.10301075268817204, \"cost\": 0.00194208, \"time\": 73.90803687572479}, \"b9bb1e6f8d\": {\"quality\": 0.06301075268817204, \"cost\": 0.012499610999999999, \"time\": 150.40043871402742}, \"b9d0e8740c\": {\"quality\": 0.10623655913978494, \"cost\": 0.034839880000000004, \"time\": 144.61329133510588}, \"b9da208432\": {\"quality\": 0.10301075268817204, \"cost\": 0.009705881, \"time\": 142.17393441200255}, \"ba3223f6ac\": {\"quality\": 0.04, \"cost\": 0.004067208, \"time\": 117.73131382465363}, \"bb13365175\": {\"quality\": 0.10946236559139785, \"cost\": 0.022399919, \"time\": 149.47980670928956}, \"bb1b3a4d29\": {\"quality\": 0.10301075268817204, \"cost\": 0.031956166, \"time\": 149.79417490959167}, \"bb6536b0ab\": {\"quality\": 0.06946236559139785, \"cost\": 0.010785974, \"time\": 102.99912581443786}, \"bbba9dd6ae\": {\"quality\": 0.11591397849462365, \"cost\": 0.005964354, \"time\": 77.64485285282134}, \"bbde69a1ae\": {\"quality\": 0.08967741935483872, \"cost\": 0.03738282200000001, \"time\": 164.79430203437806}, \"bc29a0c0fe\": {\"quality\": 0.11268817204301075, \"cost\": 0.033914098000000004, \"time\": 101.38525047302247}, \"bc3d02f753\": {\"quality\": 0.0, \"cost\": 0.0025910279999999996, \"time\": 93.81551551818848}, \"bc4c1fcc64\": {\"quality\": 0.20301075268817204, \"cost\": 0.0059535610000000004, \"time\": 67.91742668151855}, \"bd30d27f62\": {\"quality\": 0.12301075268817205, \"cost\": 0.062644566, \"time\": 121.11297037601472}, \"bd99b2fb21\": {\"quality\": 0.09333333333333334, \"cost\": 0.008204478, \"time\": 122.81714386940003}, \"bddc7d2a34\": {\"quality\": 0.10301075268817204, \"cost\": 0.03213537500000001, \"time\": 163.16414675712585}, \"be2ae88f70\": {\"quality\": 0.0, \"cost\": 0.004693302, \"time\": 137.78757257461547}, \"be4740f38f\": {\"quality\": 0.09655913978494624, \"cost\": 0.011966032000000001, \"time\": 123.94724879264831}, \"bec0c6a95f\": {\"quality\": 0.11134408602150538, \"cost\": 0.044889398, \"time\": 164.90446269512177}, \"bed888d4dc\": {\"quality\": 0.03612903225806452, \"cost\": 0.007983669, \"time\": 159.96076278686525}, \"bf45e407f6\": {\"quality\": 0.05290322580645161, \"cost\": 0.039748443999999994, \"time\": 127.84493451118469}, \"bf5550f320\": {\"quality\": 0.07333333333333333, \"cost\": 0.035326152, \"time\": 161.38907580375673}, \"bf87e58322\": {\"quality\": 0.10301075268817204, \"cost\": 0.02893854, \"time\": 130.29030165672305}, \"bfed7670ed\": {\"quality\": 0.08301075268817204, \"cost\": 0.010171252, \"time\": 128.07581346035005}, \"c0541e2220\": {\"quality\": 0.0, \"cost\": 0.037621554, \"time\": 99.32873375415802}, \"c0e10c0048\": {\"quality\": 0.0, \"cost\": 0.03382517, \"time\": 166.7884413957596}, \"c127509a7a\": {\"quality\": 0.12301075268817205, \"cost\": 0.013034976, \"time\": 122.95744087696076}, \"c13682c7c7\": {\"quality\": 0.11258064516129032, \"cost\": 0.012015295999999998, \"time\": 151.76094684600832}, \"c13d6e78e9\": {\"quality\": 0.12301075268817205, \"cost\": 0.017051258000000003, \"time\": 136.8413455247879}, \"c14ff3144d\": {\"quality\": 0.06623655913978495, \"cost\": 0.0015566399999999998, \"time\": 67.90232226848602}, \"c1e42ac47b\": {\"quality\": 0.10946236559139785, \"cost\": 0.039353246, \"time\": 155.5742819786072}, \"c2949aa902\": {\"quality\": 0.10301075268817204, \"cost\": 0.007100054999999999, \"time\": 123.65010952949524}, \"c31e956b35\": {\"quality\": 0.10301075268817204, \"cost\": 0.009654768000000001, \"time\": 160.06434454917905}, \"c36b525dde\": {\"quality\": 0.10623655913978494, \"cost\": 0.003221145, \"time\": 89.18764023780822}, \"c38326e2bd\": {\"quality\": 0.10946236559139785, \"cost\": 0.036419284, \"time\": 149.7904277563095}, \"c3ec2cec59\": {\"quality\": 0.11591397849462365, \"cost\": 0.012373450000000001, \"time\": 124.03460397720337}, \"c44720575f\": {\"quality\": 0.10623655913978494, \"cost\": 0.011866406, \"time\": 114.06694071292877}, \"c48ecefab6\": {\"quality\": 0.0, \"cost\": 0.031072968000000006, \"time\": 158.07063992023467}, \"c4a64eb40f\": {\"quality\": 0.10301075268817204, \"cost\": 0.038885108, \"time\": 112.12137053012847}, \"c4a80d19b3\": {\"quality\": 0.10301075268817204, \"cost\": 0.01095102, \"time\": 163.5285721540451}, \"c4c2826afd\": {\"quality\": 0.02258064516129032, \"cost\": 0.009101021, \"time\": 162.38105652332305}, \"c4c94a5527\": {\"quality\": 0.0, \"cost\": 0.0075986579999999995, \"time\": 122.43854382038117}, \"c4e75ee9ba\": {\"quality\": 0.14946236559139786, \"cost\": 0.06605840600000001, \"time\": 123.62291731834412}, \"c4f3e7665d\": {\"quality\": 0.04, \"cost\": 0.004755744000000001, \"time\": 128.4701939582825}, \"c5471bef57\": {\"quality\": 0.0, \"cost\": 0.034792454, \"time\": 174.61682531833648}, \"c54a408db7\": {\"quality\": 0.11729646697388633, \"cost\": 0.028817849000000003, \"time\": 119.10575988292695}, \"c59cc41335\": {\"quality\": 0.02, \"cost\": 0.029414305, \"time\": 127.53696141242982}, \"c5a0b065e0\": {\"quality\": 0.0, \"cost\": 0.011118656000000001, \"time\": 86.77302918434142}, \"c5a16b834a\": {\"quality\": 0.10623655913978494, \"cost\": 0.009676605, \"time\": 163.57068514823914}, \"c5fbe2076f\": {\"quality\": 0.09333333333333334, \"cost\": 0.016622912, \"time\": 171.14971058368684}, \"c617370f6b\": {\"quality\": 0.11134408602150538, \"cost\": 0.028784580000000004, \"time\": 123.97298481464387}, \"c67f782c7f\": {\"quality\": 0.10301075268817204, \"cost\": 0.031540366, \"time\": 122.78840267658234}, \"c691a29c42\": {\"quality\": 0.0696774193548387, \"cost\": 0.015512881999999999, \"time\": 156.41156446933746}, \"c6a339987c\": {\"quality\": 0.0, \"cost\": 0.009196331000000002, \"time\": 160.24627866744996}, \"c772ff3704\": {\"quality\": 0.06, \"cost\": 0.008646966, \"time\": 149.87581686973573}, \"c7e3f348c2\": {\"quality\": 0.10946236559139785, \"cost\": 0.03753916200000001, \"time\": 149.64094378948212}, \"c823589ab6\": {\"quality\": 0.03333333333333333, \"cost\": 0.030336296, \"time\": 124.21261048316956}, \"c82f834e85\": {\"quality\": 0.11591397849462365, \"cost\": 0.007749193, \"time\": 110.63324115276336}, \"c85099881f\": {\"quality\": 0.0032258064516129032, \"cost\": 0.00896294, \"time\": 77.65277953147888}, \"c935a33384\": {\"quality\": 0.11591397849462365, \"cost\": 0.013818304, \"time\": 147.3745896100998}, \"ca3177461f\": {\"quality\": 0.07290322580645162, \"cost\": 0.032201930000000004, \"time\": 144.48511946201324}, \"caa7c0bd6b\": {\"quality\": 0.08623655913978495, \"cost\": 0.007444665, \"time\": 109.04434747695923}, \"cac6b051e9\": {\"quality\": 0.0696774193548387, \"cost\": 0.008080419, \"time\": 151.42419612407684}, \"cacb342f64\": {\"quality\": 0.10946236559139785, \"cost\": 0.04521175200000001, \"time\": 148.37158913612365}, \"cb9948679c\": {\"quality\": 0.03612903225806452, \"cost\": 0.031414252000000004, \"time\": 155.25814893245698}, \"cbb5eb0e74\": {\"quality\": 0.10623655913978494, \"cost\": 0.008005419, \"time\": 159.4877107143402}, \"cbc32cbeff\": {\"quality\": 0.07729646697388634, \"cost\": 0.009278367, \"time\": 176.14444167613982}, \"cbd4461293\": {\"quality\": 0.09655913978494624, \"cost\": 0.0019236219999999998, \"time\": 56.308462142944336}, \"cbe2318045\": {\"quality\": 0.09602150537634407, \"cost\": 0.008356266000000001, \"time\": 120.74416449069977}, \"cc886fe337\": {\"quality\": 0.03333333333333333, \"cost\": 0.006418602, \"time\": 123.71861448287964}, \"cc9a6248a0\": {\"quality\": 0.08301075268817204, \"cost\": 0.027855250000000005, \"time\": 40.938735127449036}, \"ccb2335b3f\": {\"quality\": 0.08946236559139785, \"cost\": 0.03473691600000001, \"time\": 175.81531076431276}, \"ccdf03a55b\": {\"quality\": 0.04, \"cost\": 0.04126820400000002, \"time\": 147.74624931812286}, \"ccf72745c1\": {\"quality\": 0.11268817204301075, \"cost\": 0.010027356, \"time\": 131.06223764419556}, \"cd1d418732\": {\"quality\": 0.0, \"cost\": 0.005709352, \"time\": 72.77172949314118}, \"cd23c79db1\": {\"quality\": 0.049677419354838714, \"cost\": 0.009460574000000001, \"time\": 154.12733085155486}, \"cd64fbfcd9\": {\"quality\": 0.08623655913978495, \"cost\": 0.0068760869999999995, \"time\": 149.53058531284333}, \"cd85a01e81\": {\"quality\": 0.07333333333333333, \"cost\": 0.034051955, \"time\": 182.54483699798584}, \"ce4bc5f348\": {\"quality\": 0.0, \"cost\": 0.003405222, \"time\": 103.34760620594025}, \"ce980cf86f\": {\"quality\": 0.10301075268817204, \"cost\": 0.0039973019999999995, \"time\": 133.40581440925598}, \"ceae8b8bb9\": {\"quality\": 0.10946236559139785, \"cost\": 0.020052325000000003, \"time\": 167.7911945104599}, \"cecca90dd2\": {\"quality\": 0.10301075268817204, \"cost\": 0.004860323999999999, \"time\": 140.68442809581757}, \"cf9538faf0\": {\"quality\": 0.17052227342549925, \"cost\": 0.00948961, \"time\": 133.63166942596436}, \"cf9d2e224c\": {\"quality\": 0.06, \"cost\": 0.006049894, \"time\": 97.01887967586518}, \"cfd36f3a8c\": {\"quality\": 0.10301075268817204, \"cost\": 0.015497412000000002, \"time\": 141.9499899148941}, \"cffa29a6ef\": {\"quality\": 0.12301075268817205, \"cost\": 0.0005813769999999999, \"time\": 62.55750501155853}, \"d03596c3de\": {\"quality\": 0.10946236559139785, \"cost\": 0.007728069000000001, \"time\": 166.9729477405548}, \"d07b766487\": {\"quality\": 0.07935483870967741, \"cost\": 0.03461312800000001, \"time\": 162.30493545532227}, \"d0a0a66d75\": {\"quality\": 0.11268817204301075, \"cost\": 0.022896756000000004, \"time\": 80.08134040832519}, \"d0ce31134c\": {\"quality\": 0.05591397849462365, \"cost\": 0.038193279999999996, \"time\": 161.50320081710817}, \"d0f9633442\": {\"quality\": 0.0, \"cost\": 0.008286096, \"time\": 140.58964149951936}, \"d216eab7d8\": {\"quality\": 0.09268817204301075, \"cost\": 0.008950572, \"time\": 116.59772584438323}, \"d266c19ac8\": {\"quality\": 0.10301075268817204, \"cost\": 0.005751029999999999, \"time\": 161.90357491970062}, \"d26a70179a\": {\"quality\": 0.10623655913978494, \"cost\": 0.03584914900000001, \"time\": 172.63189315795898}, \"d2af24b59e\": {\"quality\": 0.06, \"cost\": 0.008735434, \"time\": 124.40694677829742}, \"d2f2dd5cd4\": {\"quality\": 0.13623655913978494, \"cost\": 0.023797876000000003, \"time\": 131.33156244754792}, \"d302278f85\": {\"quality\": 0.00967741935483871, \"cost\": 0.030764487000000004, \"time\": 169.61139817237853}, \"d37dcaea30\": {\"quality\": 0.10946236559139785, \"cost\": 0.019287263, \"time\": 164.99651217460632}, \"d3a2d50bd7\": {\"quality\": 0.07333333333333333, \"cost\": 0.004375113, \"time\": 134.56950817108154}, \"d3d4185487\": {\"quality\": 0.06322580645161291, \"cost\": 0.033443476, \"time\": 123.06616494655609}, \"d3db4cf84d\": {\"quality\": 0.0, \"cost\": 0.004757544, \"time\": 99.5310351371765}, \"d402233b53\": {\"quality\": 0.10946236559139785, \"cost\": 0.022382061, \"time\": 155.35112550258637}, \"d43fafa19e\": {\"quality\": 0.00967741935483871, \"cost\": 0.004969002, \"time\": 85.30777862071992}, \"d446a75eb7\": {\"quality\": 0.10946236559139785, \"cost\": 0.045008152, \"time\": 111.52242555618287}, \"d48ead13da\": {\"quality\": 0.02, \"cost\": 0.00741723, \"time\": 123.98356997966766}, \"d5016f4538\": {\"quality\": 0.0, \"cost\": 0.00508065, \"time\": 85.11343963146209}, \"d55a3613b0\": {\"quality\": 0.10623655913978494, \"cost\": 0.041770765, \"time\": 152.50331590175628}, \"d58036ba66\": {\"quality\": 0.10301075268817204, \"cost\": 0.008145371999999998, \"time\": 106.38735165596009}, \"d5a84c782e\": {\"quality\": 0.10623655913978494, \"cost\": 0.029261760000000005, \"time\": 104.74087455272675}, \"d5b2eef11c\": {\"quality\": 0.12301075268817205, \"cost\": 0.02737009, \"time\": 65.21177852153778}, \"d6040140b9\": {\"quality\": 0.10301075268817204, \"cost\": 0.014017109, \"time\": 153.3877902984619}, \"d65185c1a4\": {\"quality\": 0.10623655913978494, \"cost\": 0.00525975, \"time\": 117.94775750637055}, \"d667351f33\": {\"quality\": 0.10301075268817204, \"cost\": 0.035737883, \"time\": 160.19842824935913}, \"d6bd3b66ba\": {\"quality\": 0.04, \"cost\": 0.004024998, \"time\": 126.8981255054474}, \"d6c4e48eeb\": {\"quality\": 0.10946236559139785, \"cost\": 0.008899966, \"time\": 125.08437695503235}, \"d6cbf265ee\": {\"quality\": 0.0, \"cost\": 0.001772622, \"time\": 80.58833994865418}, \"d705447fd7\": {\"quality\": 0.02, \"cost\": 0.03513882900000001, \"time\": 158.08452146053315}, \"d73a9aab4e\": {\"quality\": 0.0, \"cost\": 0.003741966, \"time\": 86.37196826934814}, \"d752c30d07\": {\"quality\": 0.10301075268817204, \"cost\": 0.016376438, \"time\": 126.43599452972413}, \"d782682359\": {\"quality\": 0.12301075268817205, \"cost\": 0.035046282000000005, \"time\": 158.43969354629516}, \"d7c0972014\": {\"quality\": 0.07333333333333333, \"cost\": 0.010681736, \"time\": 93.32859888076783}, \"d867525748\": {\"quality\": 0.06301075268817204, \"cost\": 0.01181131, \"time\": 120.1785500049591}, \"d87eb775da\": {\"quality\": 0.10946236559139785, \"cost\": 0.008138371, \"time\": 109.30400257110595}, \"d8bab6c09b\": {\"quality\": 0.0696774193548387, \"cost\": 0.011994469, \"time\": 164.43309626579287}, \"d8bcac36e8\": {\"quality\": 0.03333333333333333, \"cost\": 0.010924840000000002, \"time\": 119.18308181762694}, \"d8eadc0190\": {\"quality\": 0.10301075268817204, \"cost\": 0.039214723, \"time\": 115.4887844324112}, \"d96677d8d4\": {\"quality\": 0.10301075268817204, \"cost\": 0.006749594999999999, \"time\": 111.51266958713532}, \"d98f22270e\": {\"quality\": 0.20301075268817204, \"cost\": 0.034043790000000004, \"time\": 121.76555292606355}, \"d9e2bb21a3\": {\"quality\": 0.05290322580645161, \"cost\": 0.005608887, \"time\": 109.89146270751954}, \"da95deeb20\": {\"quality\": 0.10301075268817204, \"cost\": 0.027880385, \"time\": 84.15849900245667}, \"daaadadcc9\": {\"quality\": 0.11591397849462365, \"cost\": 0.010063653999999998, \"time\": 151.10103166103363}, \"daf855e065\": {\"quality\": 0.11591397849462365, \"cost\": 0.006424737, \"time\": 158.07599256038665}, \"db00594832\": {\"quality\": 0.12623655913978493, \"cost\": 0.037128378000000004, \"time\": 93.38327257633209}, \"db19e677c4\": {\"quality\": 0.12301075268817205, \"cost\": 0.039896044, \"time\": 130.70764796733857}, \"db3c035639\": {\"quality\": 0.11591397849462365, \"cost\": 0.036307984, \"time\": 152.2756203889847}, \"db41487005\": {\"quality\": 0.0, \"cost\": 0.029024420000000002, \"time\": 115.28090765476227}, \"db6a7482fd\": {\"quality\": 0.11333333333333334, \"cost\": 0.027559395, \"time\": 72.33171219825743}, \"db9060cd27\": {\"quality\": 0.0696774193548387, \"cost\": 0.001964664, \"time\": 64.76092283725738}, \"dbce95a072\": {\"quality\": 0.10301075268817204, \"cost\": 0.007502654000000001, \"time\": 103.14728391170502}, \"dc195abe5e\": {\"quality\": 0.10623655913978494, \"cost\": 0.030985093999999998, \"time\": 136.5469225883484}, \"dc3f4b7138\": {\"quality\": 0.02, \"cost\": 0.028686388000000007, \"time\": 102.1283097743988}, \"dc66bccb1c\": {\"quality\": 0.11456989247311827, \"cost\": 0.006545776, \"time\": 108.06170182228088}, \"dc90065dea\": {\"quality\": 0.10301075268817204, \"cost\": 0.010758259999999999, \"time\": 109.67339315414429}, \"dd0d70fedd\": {\"quality\": 0.10301075268817204, \"cost\": 0.0019252949999999998, \"time\": 62.47777860164643}, \"dddc76b3ca\": {\"quality\": 0.12301075268817205, \"cost\": 0.03317511000000001, \"time\": 121.71198806762695}, \"de18bf45e1\": {\"quality\": 0.016129032258064516, \"cost\": 0.001520226, \"time\": 81.24462885856629}, \"de1e56370f\": {\"quality\": 0.10301075268817204, \"cost\": 0.004404816000000001, \"time\": 107.52295508384705}, \"df2160ecc8\": {\"quality\": 0.10301075268817204, \"cost\": 0.03259985, \"time\": 121.08217024803162}, \"dfda94bd2a\": {\"quality\": 0.012903225806451613, \"cost\": 0.001747728, \"time\": 94.69696862697602}, \"dff452a9ca\": {\"quality\": 0.0, \"cost\": 0.005642568000000001, \"time\": 113.53185505867005}, \"e09f75d9d6\": {\"quality\": 0.12301075268817205, \"cost\": 0.039859539, \"time\": 121.64791254997255}, \"e0b6a99753\": {\"quality\": 0.10301075268817204, \"cost\": 0.008541190000000002, \"time\": 52.27649214267731}, \"e0cf6587a7\": {\"quality\": 0.07612903225806453, \"cost\": 0.035429448, \"time\": 154.11263673305513}, \"e0de4a5929\": {\"quality\": 0.11591397849462365, \"cost\": 0.03794565500000001, \"time\": 146.6949642419815}, \"e1356fb426\": {\"quality\": 0.10301075268817204, \"cost\": 0.03224261, \"time\": 113.18122699260712}, \"e20ba014a1\": {\"quality\": 0.10623655913978494, \"cost\": 0.010716353000000001, \"time\": 146.11545753479004}, \"e21806e3bc\": {\"quality\": 0.06946236559139785, \"cost\": 0.011148466, \"time\": 108.38995244503022}, \"e24e97564c\": {\"quality\": 0.10623655913978494, \"cost\": 0.034947423000000005, \"time\": 141.45279150009156}, \"e2673c1ec8\": {\"quality\": 0.10301075268817204, \"cost\": 0.035187995, \"time\": 133.8898230791092}, \"e26c7bfbdb\": {\"quality\": 0.08301075268817204, \"cost\": 0.007581762000000001, \"time\": 99.8463338136673}, \"e2f9980b06\": {\"quality\": 0.11591397849462365, \"cost\": 0.019417651, \"time\": 133.10050780773162}, \"e3445f7632\": {\"quality\": 0.10623655913978494, \"cost\": 0.004820125, \"time\": 63.44422569274902}, \"e35e5f81a7\": {\"quality\": 0.0, \"cost\": 0.006810027, \"time\": 115.5727329492569}, \"e376ac53e7\": {\"quality\": 0.08946236559139785, \"cost\": 0.024290850000000003, \"time\": 110.75653586387634}, \"e3d8bb56da\": {\"quality\": 0.10301075268817204, \"cost\": 0.021544511000000002, \"time\": 116.27818155288696}, \"e3df4cf041\": {\"quality\": 0.0, \"cost\": 0.007794071, \"time\": 129.81809906959535}, \"e47dc3abca\": {\"quality\": 0.10623655913978494, \"cost\": 0.011515712, \"time\": 132.7535306930542}, \"e4b9d4fb41\": {\"quality\": 0.02, \"cost\": 0.007862585, \"time\": 139.48534836769102}, \"e510bda989\": {\"quality\": 0.13268817204301075, \"cost\": 0.005762856, \"time\": 33.74014439582825}, \"e517cd2222\": {\"quality\": 0.08761904761904762, \"cost\": 0.001565182, \"time\": 53.67046070098877}, \"e51b01f418\": {\"quality\": 0.07290322580645162, \"cost\": 0.005801265, \"time\": 138.57358028888703}, \"e520dfae5b\": {\"quality\": 0.10946236559139785, \"cost\": 0.006193371, \"time\": 176.72269680500028}, \"e521c9b7e4\": {\"quality\": 0.12301075268817205, \"cost\": 0.034744782, \"time\": 125.99484462738036}, \"e54097ad5d\": {\"quality\": 0.10301075268817204, \"cost\": 0.030318109000000006, \"time\": 143.14947426319122}, \"e56a16ca66\": {\"quality\": 0.10623655913978494, \"cost\": 0.0019251749999999997, \"time\": 100.41836931705475}, \"e5c4abf7ce\": {\"quality\": 0.10301075268817204, \"cost\": 0.03347102800000001, \"time\": 170.73657705783845}, \"e5d4689312\": {\"quality\": 0.0, \"cost\": 0.03450822800000001, \"time\": 180.7019939184189}, \"e62a7b27ae\": {\"quality\": 0.049677419354838714, \"cost\": 0.026221878, \"time\": 142.64135072231295}, \"e6a7aff3bc\": {\"quality\": 0.0, \"cost\": 0.032406869000000005, \"time\": 182.2341136932373}, \"e736999157\": {\"quality\": 0.04, \"cost\": 0.031729251, \"time\": 175.4835301399231}, \"e7517a8ce0\": {\"quality\": 0.12301075268817205, \"cost\": 0.034507195000000004, \"time\": 118.30208179950714}, \"e7520ca5ac\": {\"quality\": 0.12301075268817205, \"cost\": 0.03356782800000001, \"time\": 164.09410247802734}, \"e7e94ab7a5\": {\"quality\": 0.0, \"cost\": 0.001703792, \"time\": 94.30426328182222}, \"e887ddf5cc\": {\"quality\": 0.10623655913978494, \"cost\": 0.041667262, \"time\": 169.6624465703964}, \"e94fb5a295\": {\"quality\": 0.0, \"cost\": 0.033384014000000004, \"time\": 134.82464711666108}, \"ea6ecc5653\": {\"quality\": 0.06301075268817204, \"cost\": 0.007951587, \"time\": 134.84457714557647}, \"ea8bcb3ae2\": {\"quality\": 0.10301075268817204, \"cost\": 0.017331892, \"time\": 134.0905973672867}, \"ebbe8b6c4f\": {\"quality\": 0.0, \"cost\": 0.007050224, \"time\": 106.51247243881225}, \"ebdf3abff2\": {\"quality\": 0.10301075268817204, \"cost\": 0.013816473000000001, \"time\": 136.48189253807067}, \"ec55dba809\": {\"quality\": 0.10301075268817204, \"cost\": 0.03585076400000001, \"time\": 154.6976619243622}, \"ecb5f78f37\": {\"quality\": 0.0, \"cost\": 0.011340265, \"time\": 174.9997961997986}, \"ecda3d74cb\": {\"quality\": 0.02, \"cost\": 0.032141616, \"time\": 116.44101283550262}, \"ece10c0388\": {\"quality\": 0.18301075268817205, \"cost\": 0.05883078800000001, \"time\": 113.18113870620726}, \"ed6b5480a5\": {\"quality\": 0.06946236559139785, \"cost\": 0.0049282679999999995, \"time\": 77.08623633384704}, \"eda630dc85\": {\"quality\": 0.10623655913978494, \"cost\": 0.007813983, \"time\": 120.82467505931854}, \"edaaee5ed4\": {\"quality\": 0.08946236559139785, \"cost\": 0.006681024000000001, \"time\": 74.40945067405701}, \"edb2b764aa\": {\"quality\": 0.02, \"cost\": 0.013618342, \"time\": 113.58856484889984}, \"edc52339db\": {\"quality\": 0.03333333333333333, \"cost\": 0.013122086000000002, \"time\": 151.4774285554886}, \"ede7071775\": {\"quality\": 0.12301075268817205, \"cost\": 0.059439179999999994, \"time\": 100.97825186252595}, \"ee46042c5d\": {\"quality\": 0.0, \"cost\": 0.01214812, \"time\": 152.4212220430374}, \"ee68c51f73\": {\"quality\": 0.12301075268817205, \"cost\": 0.030089499999999998, \"time\": 61.54816117286683}, \"ee7b726747\": {\"quality\": 0.11591397849462365, \"cost\": 0.017031875000000002, \"time\": 160.18388693332673}, \"eec5f32da9\": {\"quality\": 0.11015360983102919, \"cost\": 0.040522985000000004, \"time\": 149.7510376214981}, \"eed40e4378\": {\"quality\": 0.10623655913978494, \"cost\": 0.03209097500000001, \"time\": 152.84840388298034}, \"eef12d478b\": {\"quality\": 0.08258064516129032, \"cost\": 0.0049949880000000006, \"time\": 113.2755955696106}, \"ef37b3e0be\": {\"quality\": 0.0, \"cost\": 0.009670806, \"time\": 178.49481089115142}, \"ef43d497f1\": {\"quality\": 0.08301075268817204, \"cost\": 0.012532439999999999, \"time\": 159.2441192626953}, \"ef4d4c4a62\": {\"quality\": 0.08623655913978495, \"cost\": 0.00850348, \"time\": 126.41268122196198}, \"ef9a651425\": {\"quality\": 0.11591397849462365, \"cost\": 0.036512095, \"time\": 164.66946573257445}, \"f0655621af\": {\"quality\": 0.02967741935483871, \"cost\": 0.0047729999999999995, \"time\": 45.447355723381044}, \"f076b4c9ae\": {\"quality\": 0.10623655913978494, \"cost\": 0.007286825, \"time\": 118.35513505935668}, \"f11eddb4ed\": {\"quality\": 0.10301075268817204, \"cost\": 0.012131963000000003, \"time\": 151.4715585231781}, \"f1408da253\": {\"quality\": 0.10301075268817204, \"cost\": 0.009555081, \"time\": 155.64059100151061}, \"f18cf41929\": {\"quality\": 0.0, \"cost\": 0.003205638, \"time\": 99.03487899303437}, \"f1aa0b0b42\": {\"quality\": 0.10946236559139785, \"cost\": 0.025626688, \"time\": 115.38884313106537}, \"f1bda127f6\": {\"quality\": 0.10301075268817204, \"cost\": 0.012302958, \"time\": 102.7573136806488}, \"f1f373e58e\": {\"quality\": 0.10623655913978494, \"cost\": 0.039605126000000004, \"time\": 118.39543704986573}, \"f2a2e91541\": {\"quality\": 0.09660522273425498, \"cost\": 0.008657139000000001, \"time\": 145.9810749053955}, \"f2c04ed1c8\": {\"quality\": 0.10301075268817204, \"cost\": 0.004977713999999999, \"time\": 68.23735935688019}, \"f2cf5db12d\": {\"quality\": 0.09268817204301075, \"cost\": 0.008181425999999999, \"time\": 120.30662734508515}, \"f366c0dd10\": {\"quality\": 0.00967741935483871, \"cost\": 0.029196041000000002, \"time\": 114.87482616901397}, \"f4303a5b4f\": {\"quality\": 0.10301075268817204, \"cost\": 0.041115134000000005, \"time\": 145.90202000141142}, \"f437481e3b\": {\"quality\": 0.21591397849462365, \"cost\": 0.012506865999999998, \"time\": 151.16193616390228}, \"f4bc6b63a7\": {\"quality\": 0.10301075268817204, \"cost\": 0.05883107500000001, \"time\": 110.05861134529113}, \"f4dc556633\": {\"quality\": 0.032903225806451615, \"cost\": 0.035164677000000005, \"time\": 165.65216555595396}, \"f4deb72db6\": {\"quality\": 0.0, \"cost\": 0.00779049, \"time\": 124.83286740779877}, \"f4ef2b9c33\": {\"quality\": 0.08623655913978495, \"cost\": 0.025671540000000003, \"time\": 110.72261786460876}, \"f566d6d6a1\": {\"quality\": 0.12301075268817205, \"cost\": 0.030296474, \"time\": 118.43880999088287}, \"f5b9a94dcc\": {\"quality\": 0.0, \"cost\": 0.00286785, \"time\": 96.0643517255783}, \"f5c27e7172\": {\"quality\": 0.029677419354838707, \"cost\": 0.003395451, \"time\": 131.81098673343658}, \"f5e53d963b\": {\"quality\": 0.11623655913978495, \"cost\": 0.0022498320000000002, \"time\": 82.10175185203552}, \"f614235c15\": {\"quality\": 0.0, \"cost\": 0.0015151140000000001, \"time\": 93.65864548683166}, \"f6546149e3\": {\"quality\": 0.10623655913978494, \"cost\": 0.034531665, \"time\": 126.04132678508759}, \"f74ec023e4\": {\"quality\": 0.11591397849462365, \"cost\": 0.011997618, \"time\": 167.2357141494751}, \"f7b048bd54\": {\"quality\": 0.10301075268817204, \"cost\": 0.008602385, \"time\": 132.47917804718017}, \"f7c4df993e\": {\"quality\": 0.10301075268817204, \"cost\": 0.007398474, \"time\": 140.5795171499252}, \"f854533145\": {\"quality\": 0.10301075268817204, \"cost\": 0.010546359, \"time\": 164.53556537628174}, \"f89b8a1930\": {\"quality\": 0.10946236559139785, \"cost\": 0.016153075, \"time\": 165.33965291976926}, \"f93d9a2693\": {\"quality\": 0.06301075268817204, \"cost\": 0.006399356999999998, \"time\": 130.61231911182404}, \"f97d91a249\": {\"quality\": 0.13591397849462367, \"cost\": 0.029466680000000002, \"time\": 128.38061220645903}, \"f99096d89c\": {\"quality\": 0.11268817204301075, \"cost\": 0.009063081, \"time\": 164.117862033844}, \"f9e8e221f3\": {\"quality\": 0.07729646697388634, \"cost\": 0.007258314, \"time\": 137.73151681423187}, \"fa38879eab\": {\"quality\": 0.07333333333333333, \"cost\": 0.013395525000000002, \"time\": 171.87790231704713}, \"fa71111570\": {\"quality\": 0.09333333333333334, \"cost\": 0.033387521, \"time\": 163.68935594558718}, \"fa7882d46b\": {\"quality\": 0.11591397849462365, \"cost\": 0.01214556, \"time\": 168.27967350482942}, \"fa906520d1\": {\"quality\": 0.10301075268817204, \"cost\": 0.012420450999999999, \"time\": 173.23661334514617}, \"faabebaa30\": {\"quality\": 0.08301075268817204, \"cost\": 0.01007883, \"time\": 177.29549469947813}, \"fb0339a7d0\": {\"quality\": 0.10623655913978494, \"cost\": 0.012603655000000002, \"time\": 175.26443803310394}, \"fb216ad6b3\": {\"quality\": 0.08623655913978495, \"cost\": 0.0027521299999999998, \"time\": 48.80933380126953}, \"fb6216880a\": {\"quality\": 0.04, \"cost\": 0.010161849, \"time\": 176.34763963222503}, \"fba499b89d\": {\"quality\": 0.03333333333333333, \"cost\": 0.034680020000000006, \"time\": 176.12596268653868}, \"fbc010a368\": {\"quality\": 0.0696774193548387, \"cost\": 0.03236535, \"time\": 140.89571504592897}, \"fbc02e2e07\": {\"quality\": 0.10946236559139785, \"cost\": 0.032320148, \"time\": 134.5744981765747}, \"fbd6c45271\": {\"quality\": 0.11591397849462365, \"cost\": 0.010223524000000001, \"time\": 171.81036722660065}, \"fc0a156e16\": {\"quality\": 0.10623655913978494, \"cost\": 0.014270385, \"time\": 127.52731266021729}, \"fc1fd5bf54\": {\"quality\": 0.0701536098310292, \"cost\": 0.005700248999999999, \"time\": 163.72950367927552}, \"fc6967a75b\": {\"quality\": 0.0, \"cost\": 0.03203869000000001, \"time\": 176.3084951877594}, \"fc73c3b0fa\": {\"quality\": 0.12301075268817205, \"cost\": 0.011811562000000001, \"time\": 53.39196453094482}, \"fce38334b2\": {\"quality\": 0.03612903225806452, \"cost\": 0.013067388, \"time\": 160.21355810165403}, \"fce5fca128\": {\"quality\": 0.15946236559139787, \"cost\": 0.038042624000000004, \"time\": 158.27728819847107}, \"fd0709359e\": {\"quality\": 0.10301075268817204, \"cost\": 0.0006582199999999999, \"time\": 41.16247012615204}, \"fd1f809d64\": {\"quality\": 0.10301075268817204, \"cost\": 0.040954353, \"time\": 135.35224254131316}, \"fd2c994a9d\": {\"quality\": 0.10946236559139785, \"cost\": 0.031750883, \"time\": 155.9396536588669}, \"fddccfbf94\": {\"quality\": 0.20301075268817204, \"cost\": 0.010760402, \"time\": 121.92865846157073}, \"fe7fa741b4\": {\"quality\": 0.11134408602150538, \"cost\": 0.016085802000000003, \"time\": 126.83495450019836}, \"fe9e1fec71\": {\"quality\": 0.12301075268817205, \"cost\": 0.033332814, \"time\": 128.354682469368}, \"fea4734c09\": {\"quality\": 0.12301075268817205, \"cost\": 0.005711668, \"time\": 87.13299584388733}, \"fef1ca27fa\": {\"quality\": 0.07913978494623655, \"cost\": 0.013074829999999999, \"time\": 140.50196795463563}, \"ff11cb6a7a\": {\"quality\": 0.08301075268817204, \"cost\": 0.012798740000000001, \"time\": 135.25200283527374}, \"ff171e34e2\": {\"quality\": 0.0, \"cost\": 0.004113531, \"time\": 106.45833990573882}, \"ff1c958e21\": {\"quality\": 0.0, \"cost\": 0.0030646650000000003, \"time\": 87.37032148838043}, \"ff8df4ace9\": {\"quality\": 0.08, \"cost\": 0.032799690000000006, \"time\": 86.2204920053482}, \"ff8e68049a\": {\"quality\": 0.10301075268817204, \"cost\": 0.006420515999999999, \"time\": 71.03863010406494}}" }, { "path": "abacus-research/biodex-revision-priors-maxquality.json", "content": "{\"00b02360ef\": {\"quality\": 0.21648684648684646, \"cost\": 0.040317356500000005, \"time\": 40.99749290347099}, \"025a41642e\": {\"quality\": 0.2323391192141192, \"cost\": 0.011047775499999999, \"time\": 38.92555040717125}, \"03c4bc9bb0\": {\"quality\": 0.25034264346764346, \"cost\": 0.011205506, \"time\": 22.393556386232376}, \"04ed954ae9\": {\"quality\": 0.25250624375624375, \"cost\": 0.04878718750000001, \"time\": 20.034630668163302}, \"059dee8af9\": {\"quality\": 0.2151956376956377, \"cost\": 0.00895401925, \"time\": 31.44526737332344}, \"05fde83b5e\": {\"quality\": 0.20102897102897105, \"cost\": 0.007020948000000001, \"time\": 27.97608198523521}, \"06b62ff472\": {\"quality\": 0.26966144966144967, \"cost\": 0.005215445, \"time\": 14.247572779655457}, \"0730a1221a\": {\"quality\": 0.25292291042291043, \"cost\": 0.027662595, \"time\": 33.58496024608612}, \"07aec969dd\": {\"quality\": 0.2148887223887224, \"cost\": 0.0008298215000000001, \"time\": 43.21092504262924}, \"0a11f0ae2e\": {\"quality\": 0.24079150016650014, \"cost\": 0.010170454500000002, \"time\": 20.595997565984725}, \"0aeb79b24c\": {\"quality\": 0.14777319902319902, \"cost\": 0.0058596425, \"time\": 7.766067373752594}, \"0b3eafcb23\": {\"quality\": 0.1913265900765901, \"cost\": 0.0013463715, \"time\": 19.126238137483597}, \"0bc5bbe9f8\": {\"quality\": 0.19658452658452658, \"cost\": 0.0314150555, \"time\": 42.02523586153984}, \"0c328ddadf\": {\"quality\": 0.1950774919524919, \"cost\": 0.007442179, \"time\": 22.943194967508315}, \"0db4d3e16d\": {\"quality\": 0.18679285991785993, \"cost\": 0.0032097465, \"time\": 19.68699924945831}, \"0df97ab2f2\": {\"quality\": 0.22131729381729381, \"cost\": 0.008563199, \"time\": 43.75024158954621}, \"0e0162da5c\": {\"quality\": 0.21570068820068822, \"cost\": 0.027131577500000004, \"time\": 33.71510918140412}, \"0e0216352f\": {\"quality\": 0.16231865356865358, \"cost\": 0.009209249, \"time\": 33.987769949436185}, \"0ecd21b93a\": {\"quality\": 0.2128450715950716, \"cost\": 0.027050046, \"time\": 27.799335622787474}, \"0ff2bde030\": {\"quality\": 0.2117928599178599, \"cost\": 0.033819734000000004, \"time\": 19.09183721542358}, \"1059f21a6c\": {\"quality\": 0.21909507159507158, \"cost\": 0.0006545234999999999, \"time\": 13.647678703069687}, \"112b665311\": {\"quality\": 0.21884552947052946, \"cost\": 0.004884652, \"time\": 22.208240193128585}, \"11db1dbc4f\": {\"quality\": 0.1719506882006882, \"cost\": 0.003233838, \"time\": 22.85480940937996}, \"13145c10ec\": {\"quality\": 0.23070159007659008, \"cost\": 0.0313856595, \"time\": 35.557817763090135}, \"147915874a\": {\"quality\": 0.2280723443223443, \"cost\": 0.00055762025, \"time\": 34.335060054063796}, \"16e1e2aca6\": {\"quality\": 0.25766650016650017, \"cost\": 0.005274602500000001, \"time\": 13.559284949302672}, \"1914dfad94\": {\"quality\": 0.2624284049284049, \"cost\": 0.02536484, \"time\": 25.639233177900316}, \"1ab9dcd7d6\": {\"quality\": 0.2867339604839605, \"cost\": 0.0010822935, \"time\": 18.924210703372957}, \"1dc749411d\": {\"quality\": 0.2199178599178599, \"cost\": 0.010029340000000001, \"time\": 14.274628734588624}, \"1dd84b0308\": {\"quality\": 0.2286371961371961, \"cost\": 0.027319404999999998, \"time\": 25.991028892993928}, \"1dd9d9bef2\": {\"quality\": 0.2226475607725608, \"cost\": 0.007434026250000001, \"time\": 31.69112855195999}, \"1f0ba92b8b\": {\"quality\": 0.22614052614052613, \"cost\": 0.0175957475, \"time\": 36.91505531668663}, \"1f1b5a019d\": {\"quality\": 0.2593331668331669, \"cost\": 0.033523670000000005, \"time\": 26.298819702863693}, \"1f41a35dd3\": {\"quality\": 0.14541666666666667, \"cost\": 0.004300389, \"time\": 17.449908739328386}, \"212c3fccbf\": {\"quality\": 0.2939645770895771, \"cost\": 0.0327319625, \"time\": 27.72252732515335}, \"219d51cd03\": {\"quality\": 0.1703336247086247, \"cost\": 0.0009408769999999999, \"time\": 18.91644185781479}, \"238ed27a56\": {\"quality\": 0.24007659007659007, \"cost\": 0.0295427895, \"time\": 31.712434768676758}, \"23f3d97d47\": {\"quality\": 0.23146457708957707, \"cost\": 0.02700624675, \"time\": 32.10481671690941}, \"253547264d\": {\"quality\": 0.1917432567432567, \"cost\": 0.026901585, \"time\": 23.115290719270703}, \"27e14a0973\": {\"quality\": 0.23250624375624376, \"cost\": 0.033178285, \"time\": 36.78056996464729}, \"29960739d3\": {\"quality\": 0.221773643023643, \"cost\": 0.032106853000000005, \"time\": 29.176937651634216}, \"2b0a8d5614\": {\"quality\": 0.2043530081030081, \"cost\": 0.022987428499999997, \"time\": 20.08997523188591}, \"2ccd104a68\": {\"quality\": 0.20102897102897105, \"cost\": 0.0184998355, \"time\": 45.19519582390785}, \"2d1b0cd5d9\": {\"quality\": 0.22214008214008213, \"cost\": 0.0059833375, \"time\": 18.541752833127973}, \"2dcffa3d30\": {\"quality\": 0.27596042846042845, \"cost\": 0.009521012499999999, \"time\": 26.920003366470336}, \"2ebe033e6a\": {\"quality\": 0.21273532023532027, \"cost\": 0.0073467315, \"time\": 21.433203184604643}, \"31e8ef9410\": {\"quality\": 0.10839285714285715, \"cost\": 0.0007289739999999999, \"time\": 63.23130738735199}, \"32a628c971\": {\"quality\": 0.25402500277500273, \"cost\": 0.037233726, \"time\": 26.05542423725128}, \"336e1a4fdc\": {\"quality\": 0.1680573593073593, \"cost\": 0.031870938, \"time\": 38.903397005796435}, \"33e0fdf4e1\": {\"quality\": 0.2249284049284049, \"cost\": 0.005128618, \"time\": 22.448986053466797}, \"38a60c768a\": {\"quality\": 0.23622648185148185, \"cost\": 0.027102593, \"time\": 38.203392094373704}, \"39bb54fbfb\": {\"quality\": 0.248528971028971, \"cost\": 0.03168909524999999, \"time\": 49.03234923481941}, \"3ace395562\": {\"quality\": 0.20436230436230435, \"cost\": 0.025813554500000002, \"time\": 24.070003932714464}, \"3bbe08bf63\": {\"quality\": 0.2427335164835165, \"cost\": 0.033579119000000004, \"time\": 43.73568968772888}, \"3caf8d77c5\": {\"quality\": 0.25665397102897103, \"cost\": 0.0361807135, \"time\": 33.51353464722634}, \"3d25abc0cb\": {\"quality\": 0.2298871961371961, \"cost\": 0.008877388500000001, \"time\": 38.62544178366661}, \"3e19290ef5\": {\"quality\": 0.2251261932511933, \"cost\": 0.026017763000000006, \"time\": 31.622546792030334}, \"3e8e089a02\": {\"quality\": 0.232903971028971, \"cost\": 0.0243675525, \"time\": 12.379487264156342}, \"3f1771caee\": {\"quality\": 0.1291142884892885, \"cost\": 0.003903147, \"time\": 22.171902561187743}, \"3f7beb53c4\": {\"quality\": 0.26968038905538905, \"cost\": 0.00175834875, \"time\": 17.588406985998155}, \"4081f36f1d\": {\"quality\": 0.20352897102897105, \"cost\": 0.009008155, \"time\": 35.7961721599102}, \"410edf5f69\": {\"quality\": 0.24408452658452662, \"cost\": 0.033720045500000004, \"time\": 39.4614034473896}, \"44ec78f301\": {\"quality\": 0.2112887112887113, \"cost\": 0.0312310875, \"time\": 28.79513317346573}, \"44eeb11408\": {\"quality\": 0.23811230436230435, \"cost\": 0.027474213500000004, \"time\": 33.9330511868}, \"45623e6def\": {\"quality\": 0.27609841547341546, \"cost\": 0.0343063295, \"time\": 33.21763527989388}, \"470749179f\": {\"quality\": 0.22933316683316685, \"cost\": 0.0050279755, \"time\": 24.99732105731964}, \"47ea56eba0\": {\"quality\": 0.19954240204240203, \"cost\": 0.018521998, \"time\": 30.92959639430046}, \"49fc8b3768\": {\"quality\": 0.16649433899433902, \"cost\": 0.0028119045000000002, \"time\": 17.623043191432952}, \"4a60adbf47\": {\"quality\": 0.18713078588078588, \"cost\": 0.013557406999999999, \"time\": 28.078510254621506}, \"4ba372d292\": {\"quality\": 0.24674478299478303, \"cost\": 0.007506609250000001, \"time\": 32.822478234767914}, \"4bf38b1922\": {\"quality\": 0.19991785991785993, \"cost\": 0.031763846750000005, \"time\": 40.722243314981455}, \"4c415b4b86\": {\"quality\": 0.1526658757908758, \"cost\": 0.025909743, \"time\": 14.397501182556152}, \"4d1fc31b49\": {\"quality\": 0.2615845265845266, \"cost\": 0.024915399, \"time\": 18.815959566831587}, \"4d8c84f53a\": {\"quality\": 0.2150567488067488, \"cost\": 0.0271872135, \"time\": 31.158079880476}, \"4df2245b9b\": {\"quality\": 0.14843878343878342, \"cost\": 0.0011828115, \"time\": 5.0212810933589935}, \"4e9274d39b\": {\"quality\": 0.2178261322011322, \"cost\": 0.0059426750000000006, \"time\": 16.452285903692246}, \"4f87ed7c2e\": {\"quality\": 0.23269563769563767, \"cost\": 0.0090589945, \"time\": 26.12176838517189}, \"51a340bebc\": {\"quality\": 0.23375624375624376, \"cost\": 0.032705517, \"time\": 28.51812062859535}, \"52dee74e25\": {\"quality\": 0.2215728021978022, \"cost\": 0.032394368, \"time\": 31.705753284692765}, \"533fddb3bd\": {\"quality\": 0.13879745254745254, \"cost\": 0.0016485569999999997, \"time\": 9.408051878213882}, \"5358cb5855\": {\"quality\": 0.18664467476967478, \"cost\": 0.027746372000000002, \"time\": 23.547580116987227}, \"53b6c3e00f\": {\"quality\": 0.2590950715950716, \"cost\": 0.000778896, \"time\": 12.53750283718109}, \"5400c14640\": {\"quality\": 0.2697979104229104, \"cost\": 0.029639904999999994, \"time\": 41.52563037276268}, \"55e373a7e2\": {\"quality\": 0.16850885225885226, \"cost\": 0.0406613025, \"time\": 22.814987272024155}, \"5652070ecb\": {\"quality\": 0.17858849483849482, \"cost\": 0.0075221129999999995, \"time\": 32.25966700911522}, \"56c5cbc75a\": {\"quality\": 0.23478951603951606, \"cost\": 0.000632159, \"time\": 38.25331119894982}, \"577b168d21\": {\"quality\": 0.2182725607725608, \"cost\": 0.00705232, \"time\": 13.326130753755569}, \"57cabf0e4b\": {\"quality\": 0.22299325674325673, \"cost\": 0.02575083125, \"time\": 24.347850561141968}, \"58dbfc0499\": {\"quality\": 0.2620895770895771, \"cost\": 0.055131649000000005, \"time\": 28.987264293432233}, \"5a5739c3d2\": {\"quality\": 0.20117597680097682, \"cost\": 0.007228886499999999, \"time\": 25.403514659404756}, \"5b12979e52\": {\"quality\": 0.18208513708513707, \"cost\": 0.0038426475, \"time\": 22.1737478017807}, \"5b384da665\": {\"quality\": 0.20099920912420913, \"cost\": 0.014891756499999999, \"time\": 28.11651138663292}, \"5b837baf2e\": {\"quality\": 0.23083957708957709, \"cost\": 0.04059063350000001, \"time\": 38.7564038336277}, \"5c8acd9a75\": {\"quality\": 0.1964952408702409, \"cost\": 0.027179780499999997, \"time\": 31.81220202445984}, \"5da68874af\": {\"quality\": 0.1768232461982462, \"cost\": 0.00942769325, \"time\": 30.90108055472374}, \"5e304846b6\": {\"quality\": 0.13062375124875125, \"cost\": 0.00801055675, \"time\": 21.377113670110703}, \"5e608d2a8b\": {\"quality\": 0.160135212010212, \"cost\": 0.0008863050000000001, \"time\": 21.419854152202603}, \"5e6ad9fb8a\": {\"quality\": 0.19750624375624376, \"cost\": 0.025521787500000004, \"time\": 20.569793158769606}, \"5ec5b185c9\": {\"quality\": 0.18449037074037078, \"cost\": 0.002478051, \"time\": 34.97374950647354}, \"608e332141\": {\"quality\": 0.24616785991785994, \"cost\": 0.03769494400000001, \"time\": 25.56632845401764}, \"610753ffe5\": {\"quality\": 0.2546202408702409, \"cost\": 0.0301900835, \"time\": 22.38964868783951}, \"614e59c7e7\": {\"quality\": 0.22919427794427794, \"cost\": 0.0017370554999999999, \"time\": 16.915371453762056}, \"62320710f0\": {\"quality\": 0.25123730436230435, \"cost\": 0.03658336050000001, \"time\": 31.835360258817673}, \"649df27d73\": {\"quality\": 0.23370664058164056, \"cost\": 0.033310952000000005, \"time\": 22.02554016113281}, \"64f956ba72\": {\"quality\": 0.2440845265845266, \"cost\": 0.028164125, \"time\": 15.42359745502472}, \"65657a3d04\": {\"quality\": 0.21516650016650019, \"cost\": 0.011299091, \"time\": 48.3486190378666}, \"6604abbd43\": {\"quality\": 0.24808316683316683, \"cost\": 0.026956945250000003, \"time\": 36.97830517292023}, \"67050fa89b\": {\"quality\": 0.18341144966144968, \"cost\": 0.0313915525, \"time\": 39.6457058608532}, \"67f05c5985\": {\"quality\": 0.22079150016650018, \"cost\": 0.02690715625, \"time\": 22.91822772026062}, \"67fe27bce0\": {\"quality\": 0.24436230436230436, \"cost\": 0.041374972, \"time\": 33.75530356168747}, \"698d1b1d0e\": {\"quality\": 0.25665397102897103, \"cost\": 0.04129188500000001, \"time\": 50.97259179949761}, \"69e8aa9f73\": {\"quality\": 0.1903450715950716, \"cost\": 0.00740056, \"time\": 34.38110066056252}, \"6b240a8971\": {\"quality\": 0.21965180652680652, \"cost\": 0.0090931325, \"time\": 28.12567190527916}, \"6b56430005\": {\"quality\": 0.23343968531468534, \"cost\": 0.027798901, \"time\": 43.157285338640214}, \"6b611b7193\": {\"quality\": 0.2807205294705295, \"cost\": 0.06928643750000002, \"time\": 21.161213809251784}, \"6b77ef93d3\": {\"quality\": 0.1493911643911644, \"cost\": 0.0015440895, \"time\": 12.580836659669876}, \"6beefce17b\": {\"quality\": 0.27009552947052945, \"cost\": 0.03215786600000001, \"time\": 23.689218693971632}, \"6cb7c9a802\": {\"quality\": 0.27246836496836496, \"cost\": 0.0316063565, \"time\": 41.612704527378085}, \"6cba1da5b3\": {\"quality\": 0.18273684648684646, \"cost\": 0.013787275500000001, \"time\": 24.630589830875394}, \"6dea54bf99\": {\"quality\": 0.2606321456321456, \"cost\": 0.004957138, \"time\": 21.090058833360672}, \"6eda6f7780\": {\"quality\": 0.25468038905538903, \"cost\": 0.011272949, \"time\": 26.375605469942094}, \"708f3ca5da\": {\"quality\": 0.12009552947052946, \"cost\": 0.0015416722499999999, \"time\": 13.809926038980484}, \"72d75a5f01\": {\"quality\": 0.23014513264513264, \"cost\": 0.0153489635, \"time\": 28.418202906847}, \"72d99dee93\": {\"quality\": 0.2350756882006882, \"cost\": 0.029338768500000008, \"time\": 44.55438640117645}, \"73272e5bd3\": {\"quality\": 0.2751956376956377, \"cost\": 0.0333353575, \"time\": 29.94201866388321}, \"74608570f5\": {\"quality\": 0.11613782051282051, \"cost\": 0.0005538545, \"time\": 27.12369663715363}, \"7531840182\": {\"quality\": 0.1604120879120879, \"cost\": 0.0019984210000000002, \"time\": 58.24117745161056}, \"77ba656673\": {\"quality\": 0.18522602397602397, \"cost\": 0.008878578250000001, \"time\": 24.636592674255372}, \"77fe41f02f\": {\"quality\": 0.1971118464868465, \"cost\": 0.04380676650000001, \"time\": 36.06331704854965}, \"7857e95756\": {\"quality\": 0.2174590687090687, \"cost\": 0.014965390000000002, \"time\": 26.90245589017868}, \"78984e94f0\": {\"quality\": 0.21357017982017984, \"cost\": 0.007025722, \"time\": 20.596780753135683}, \"7986633543\": {\"quality\": 0.20898684648684648, \"cost\": 0.00747764175, \"time\": 33.76270450353623}, \"7bdbc32b57\": {\"quality\": 0.27164314851814847, \"cost\": 0.04712992375, \"time\": 27.709715336561203}, \"7c89cff787\": {\"quality\": 0.15383470695970697, \"cost\": 0.0016299584999999999, \"time\": 19.46316229701042}, \"7dc51ec191\": {\"quality\": 0.2680617993117993, \"cost\": 0.041678240500000005, \"time\": 52.64214360117913}, \"7e4a6245ac\": {\"quality\": 0.19536401098901102, \"cost\": 0.00056989225, \"time\": 64.66312664747238}, \"7ebc55ebcf\": {\"quality\": 0.26186230436230434, \"cost\": 0.0089994125, \"time\": 49.139087688922885}, \"7f14114a48\": {\"quality\": 0.20424325674325672, \"cost\": 0.0130888145, \"time\": 24.788931846618652}, \"7fb587503e\": {\"quality\": 0.2053228021978022, \"cost\": 0.009328153000000002, \"time\": 19.42089232802391}, \"7ff82b6f8d\": {\"quality\": 0.20713474025974027, \"cost\": 0.029847356000000005, \"time\": 21.508908247947694}, \"806a5ef096\": {\"quality\": 0.22686230436230437, \"cost\": 0.024261581, \"time\": 14.832676833868026}, \"8156d78e42\": {\"quality\": 0.1876665001665002, \"cost\": 0.007510368999999999, \"time\": 39.953574234247206}, \"815cfb848c\": {\"quality\": 0.24188124375624376, \"cost\": 0.02595560125, \"time\": 23.38996571302414}, \"820b42e0b1\": {\"quality\": 0.22338078588078586, \"cost\": 0.0027539085000000004, \"time\": 21.940480226278304}, \"82a733626e\": {\"quality\": 0.24082063769563766, \"cost\": 0.029802983, \"time\": 24.7150899887085}, \"83d6300ced\": {\"quality\": 0.22958957708957708, \"cost\": 0.009135905500000001, \"time\": 44.96203254461288}, \"844c822d56\": {\"quality\": 0.19127851315351316, \"cost\": 0.0269840145, \"time\": 23.020123237371443}, \"84f9f20f75\": {\"quality\": 0.2272988122988123, \"cost\": 0.02394343, \"time\": 14.9634037733078}, \"8525986a00\": {\"quality\": 0.2595895770895771, \"cost\": 0.03125997600000001, \"time\": 45.89759058356285}, \"85db012139\": {\"quality\": 0.28459047896547895, \"cost\": 0.04838826250000001, \"time\": 21.917299997806552}, \"85ddc8a71e\": {\"quality\": 0.19811230436230437, \"cost\": 0.00595452, \"time\": 21.147695326805113}, \"8767267bed\": {\"quality\": 0.1887950244200244, \"cost\": 0.0064469125000000006, \"time\": 7.573464637994766}, \"882238f677\": {\"quality\": 0.1552930402930403, \"cost\": 0.0032291700000000004, \"time\": 25.413773769140242}, \"88f78bdeaf\": {\"quality\": 0.12028311965811966, \"cost\": 0.0005936184999999999, \"time\": 24.23035447001457}, \"8af1605300\": {\"quality\": 0.21607017982017981, \"cost\": 0.007401837500000002, \"time\": 26.804660773277284}, \"8c09cba5e2\": {\"quality\": 0.21450119325119324, \"cost\": 0.02589227425, \"time\": 26.046354007720947}, \"8e5d966cf6\": {\"quality\": 0.15911401098901098, \"cost\": 0.0016278529999999999, \"time\": 76.43126168251038}, \"8eacbc7240\": {\"quality\": 0.2039868464868465, \"cost\": 0.0132405425, \"time\": 26.23775848150253}, \"8fc917c575\": {\"quality\": 0.2181123043623044, \"cost\": 0.0075467905, \"time\": 27.82010214924812}, \"8fe43ec148\": {\"quality\": 0.24137862137862137, \"cost\": 0.0031192755, \"time\": 28.713243955373763}, \"900de152da\": {\"quality\": 0.2329229104229104, \"cost\": 0.028014499999999998, \"time\": 11.612605232000352}, \"901ab65b55\": {\"quality\": 0.2703345265845266, \"cost\": 0.027957687500000005, \"time\": 9.624877899885178}, \"918182aa18\": {\"quality\": 0.19238629426129428, \"cost\": 0.002521713, \"time\": 45.21123977303505}, \"91ac37a21a\": {\"quality\": 0.140364010989011, \"cost\": 0.0072078985000000005, \"time\": 59.984852832555774}, \"9252afa4ec\": {\"quality\": 0.20398532023532023, \"cost\": 0.026106901500000005, \"time\": 17.68542433977127}, \"939b0c3ed9\": {\"quality\": 0.2067141192141192, \"cost\": 0.009995370500000001, \"time\": 20.511219489574433}, \"93a705cb0d\": {\"quality\": 0.23775578588078589, \"cost\": 0.0107520785, \"time\": 34.44544678330421}, \"94091e2968\": {\"quality\": 0.2271605477855478, \"cost\": 0.011070436, \"time\": 29.48041752576828}, \"945a9565fa\": {\"quality\": 0.19553356365856367, \"cost\": 0.005568486, \"time\": 9.69417832493782}, \"946baa620a\": {\"quality\": 0.2275567488067488, \"cost\": 0.025270525000000002, \"time\": 17.454011952877046}, \"950ed976dd\": {\"quality\": 0.1091089466089466, \"cost\": 0.00364847625, \"time\": 27.579946410655978}, \"955689ee8f\": {\"quality\": 0.18228708791208792, \"cost\": 0.001921847, \"time\": 29.050129276514056}, \"95c9b1b1dd\": {\"quality\": 0.21440018315018317, \"cost\": 0.031808489, \"time\": 30.75392867922783}, \"95ebba630d\": {\"quality\": 0.16167443667443665, \"cost\": 0.0045665385, \"time\": 21.370410311222074}, \"966282f059\": {\"quality\": 0.16554410866910865, \"cost\": 0.023086932000000004, \"time\": 19.98903277516365}, \"96c401772b\": {\"quality\": 0.24861825674325677, \"cost\": 0.0028384205, \"time\": 21.471487557888032}, \"9718d880f6\": {\"quality\": 0.1984595265845266, \"cost\": 0.0316786545, \"time\": 50.695869612693784}, \"971ae914cb\": {\"quality\": 0.2652461427461428, \"cost\": 0.025022515000000002, \"time\": 14.41015980243683}, \"97566e70dd\": {\"quality\": 0.23030386280386278, \"cost\": 0.0172863905, \"time\": 32.660637366771695}, \"97c882ca1f\": {\"quality\": 0.22630674880674884, \"cost\": 0.036967587999999996, \"time\": 26.217366951704026}, \"98e28fbb93\": {\"quality\": 0.1839868464868465, \"cost\": 0.0100201785, \"time\": 10.730712401866914}, \"992ab25916\": {\"quality\": 0.23011779886779885, \"cost\": 0.00176559, \"time\": 17.568689429759978}, \"9a271dff29\": {\"quality\": 0.19818986568986569, \"cost\": 0.0093025505, \"time\": 29.283438795804976}, \"9aa6fd8b03\": {\"quality\": 0.20840992340992343, \"cost\": 0.0059763937500000005, \"time\": 14.063047587871552}, \"9b12b7d441\": {\"quality\": 0.26338078588078584, \"cost\": 0.01091874575, \"time\": 34.633186584711076}, \"9bf31a2127\": {\"quality\": 0.20186230436230435, \"cost\": 0.0335034275, \"time\": 53.72036165595055}, \"9c4ad5aed3\": {\"quality\": 0.21813124375624376, \"cost\": 0.025749503750000007, \"time\": 27.629801028966902}, \"9cbb34ee5f\": {\"quality\": 0.19157370407370405, \"cost\": 0.0024265880000000004, \"time\": 20.018360090255737}, \"9e16d7ba9c\": {\"quality\": 0.2621788628038628, \"cost\": 0.032505795500000004, \"time\": 34.68335065841675}, \"9e1a7dd196\": {\"quality\": 0.19222097347097347, \"cost\": 0.003654598499999999, \"time\": 18.135820919275282}, \"9e6775ac33\": {\"quality\": 0.1184478021978022, \"cost\": 0.0016337835, \"time\": 15.268957149982452}, \"9e8674b6c7\": {\"quality\": 0.2206123043623044, \"cost\": 0.033377518, \"time\": 39.97827153205871}, \"9ea12d8d80\": {\"quality\": 0.15865613553113553, \"cost\": 0.0066404640000000004, \"time\": 11.089978641271593}, \"a232f1dfb6\": {\"quality\": 0.23243922743922746, \"cost\": 0.0017390729999999998, \"time\": 17.556494808197023}, \"a3b7cb6c33\": {\"quality\": 0.22440018315018312, \"cost\": 0.0092955765, \"time\": 25.77978389263153}, \"a72f24f687\": {\"quality\": 0.22096764346764347, \"cost\": 0.008759131000000002, \"time\": 17.248067778348922}, \"a8aac20fd9\": {\"quality\": 0.24810057997558, \"cost\": 0.028210719500000002, \"time\": 20.938565105199814}, \"a9e811d4a7\": {\"quality\": 0.2364456376956377, \"cost\": 0.024409971, \"time\": 15.286300283670425}, \"aa1fc47a86\": {\"quality\": 0.20214008214008214, \"cost\": 0.02572883575, \"time\": 27.581201010942458}, \"aa67b102e7\": {\"quality\": 0.23873730436230436, \"cost\": 0.030387160750000003, \"time\": 30.50166518688202}, \"ad453a813f\": {\"quality\": 0.21407370407370407, \"cost\": 0.003257968, \"time\": 14.862909889221193}, \"add25d67a0\": {\"quality\": 0.21374875124875123, \"cost\": 0.028111017000000002, \"time\": 24.08863323330879}, \"adf6ae1ba7\": {\"quality\": 0.16509552947052947, \"cost\": 0.0030274415000000002, \"time\": 9.59496791958809}, \"af57afe626\": {\"quality\": 0.238234681984682, \"cost\": 0.0028139635, \"time\": 19.57368974685669}, \"b07ae35700\": {\"quality\": 0.220353465978466, \"cost\": 0.003183216, \"time\": 24.758755880594254}, \"b09741c8f7\": {\"quality\": 0.1432112332112332, \"cost\": 0.0005155585, \"time\": 36.59446266293526}, \"b1b81c0847\": {\"quality\": 0.2774873043623044, \"cost\": 0.024767306250000003, \"time\": 17.89712796807289}, \"b38fbeda99\": {\"quality\": 0.19523684648684647, \"cost\": 0.00822793925, \"time\": 30.255630522966385}, \"b3b92b1835\": {\"quality\": 0.26213078588078587, \"cost\": 0.04786767, \"time\": 23.704857051372528}, \"b3bba3eee2\": {\"quality\": 0.1911576617826618, \"cost\": 0.009401211, \"time\": 32.60036996603012}, \"b40727bd68\": {\"quality\": 0.2542234154734155, \"cost\": 0.025418039000000003, \"time\": 17.466598081588742}, \"b6a0f78896\": {\"quality\": 0.23056179931179932, \"cost\": 0.008894759499999998, \"time\": 32.4968527674675}, \"b84e122880\": {\"quality\": 0.24589008214008212, \"cost\": 0.008891399000000001, \"time\": 39.84640188813209}, \"b8c6c44a39\": {\"quality\": 0.24308316683316683, \"cost\": 0.029864208750000003, \"time\": 21.440666025877}, \"bb9ed9dffe\": {\"quality\": 0.15225163725163726, \"cost\": 0.0015893925, \"time\": 19.59902848601341}, \"bbc50a0411\": {\"quality\": 0.1945796564546564, \"cost\": 0.0031702530000000005, \"time\": 33.99009187817573}, \"bbd0498616\": {\"quality\": 0.22329150016650018, \"cost\": 0.05419105850000001, \"time\": 20.79185708165169}, \"bd018ec9d7\": {\"quality\": 0.20710851648351647, \"cost\": 0.0006870085, \"time\": 58.566940271854406}, \"be11a8a86e\": {\"quality\": 0.22035589410589412, \"cost\": 0.014606052500000001, \"time\": 33.792596000432965}, \"bfa93a259c\": {\"quality\": 0.25311230436230436, \"cost\": 0.02601113875, \"time\": 29.902825361490248}, \"c127942d8c\": {\"quality\": 0.2601658757908758, \"cost\": 0.0032753834999999995, \"time\": 25.132321882247922}, \"c14216d744\": {\"quality\": 0.23213078588078587, \"cost\": 0.01049051375, \"time\": 26.48539803624153}, \"c2f0bc6921\": {\"quality\": 0.15319229381729382, \"cost\": 0.0006038535000000001, \"time\": 16.494534188508986}, \"c440b67f31\": {\"quality\": 0.19924325674325674, \"cost\": 0.030812481500000002, \"time\": 25.34757208228111}, \"caa29fbe8c\": {\"quality\": 0.229778971028971, \"cost\": 0.025535755, \"time\": 26.289914256334306}, \"cd46331132\": {\"quality\": 0.2289372433122433, \"cost\": 0.0059953365, \"time\": 16.957564985752107}, \"cdd01242f5\": {\"quality\": 0.23936230436230438, \"cost\": 0.030000593500000002, \"time\": 23.897997051477432}, \"cdf27f19d3\": {\"quality\": 0.21152472527472527, \"cost\": 0.00597143625, \"time\": 21.69409868121147}, \"ce7d236454\": {\"quality\": 0.21988629426129425, \"cost\": 0.0319162425, \"time\": 34.63264524936676}, \"d117c8e23b\": {\"quality\": 0.24436230436230436, \"cost\": 0.04203393050000001, \"time\": 37.38888673782348}, \"d27a5ad38a\": {\"quality\": 0.19329240204240206, \"cost\": 0.025790743000000005, \"time\": 19.247011250257493}, \"d287ceaae7\": {\"quality\": 0.26063124375624375, \"cost\": 0.036845655000000005, \"time\": 33.75713464021683}, \"d29027747a\": {\"quality\": 0.12741758241758241, \"cost\": 0.0004609135, \"time\": 47.16611239314079}, \"d2d7780d31\": {\"quality\": 0.2296599234099234, \"cost\": 0.033220648000000005, \"time\": 29.299119901657104}, \"d566a94f3a\": {\"quality\": 0.2568127011877012, \"cost\": 0.023108875, \"time\": 7.455539703369141}, \"d64dd29abb\": {\"quality\": 0.2582205294705294, \"cost\": 0.039288317, \"time\": 35.80952010750771}, \"d75495aed3\": {\"quality\": 0.24019563769563768, \"cost\": 0.0183352845, \"time\": 36.634383380413055}, \"d93337a87d\": {\"quality\": 0.24102897102897103, \"cost\": 0.010006869500000001, \"time\": 21.931237626075745}, \"de08b84b51\": {\"quality\": 0.23914557664557662, \"cost\": 0.005503559999999999, \"time\": 15.377897375822068}, \"de39f4f113\": {\"quality\": 0.19986825674325676, \"cost\": 0.047524011, \"time\": 18.598653775453567}, \"de83615ac5\": {\"quality\": 0.27044275169275167, \"cost\": 0.00756211075, \"time\": 28.75660304427147}, \"dfa1e5267d\": {\"quality\": 0.2558807858807859, \"cost\": 0.0013494974999999998, \"time\": 9.088204801082611}, \"dfedfba6df\": {\"quality\": 0.2490950715950716, \"cost\": 0.004502494499999999, \"time\": 26.490580713748933}, \"e15d59fc9d\": {\"quality\": 0.2783206376956377, \"cost\": 0.011371385500000001, \"time\": 52.64762927293778}, \"e540f66a9d\": {\"quality\": 0.1645820845820846, \"cost\": 0.025751229500000007, \"time\": 19.154958713054658}, \"e5c7c9fce2\": {\"quality\": 0.24218129093129093, \"cost\": 0.0258564275, \"time\": 28.940241599082945}, \"e73beb7df9\": {\"quality\": 0.1834839466089466, \"cost\": 0.02164974225, \"time\": 23.38083545565605}, \"e939aaa8b3\": {\"quality\": 0.21797341547341548, \"cost\": 0.028107724500000004, \"time\": 34.55525960922241}, \"e9e55ba5db\": {\"quality\": 0.1835701798201798, \"cost\": 0.02538775525, \"time\": 25.480451303720475}, \"ea817a074f\": {\"quality\": 0.1976956376956377, \"cost\": 0.031266095, \"time\": 24.096565490961076}, \"eca3d11e40\": {\"quality\": 0.1920706376956377, \"cost\": 0.0259032935, \"time\": 21.720754396915435}, \"ed945b7e4a\": {\"quality\": 0.21056270118770118, \"cost\": 0.03181901100000001, \"time\": 23.55217906832695}, \"ee4ddebe70\": {\"quality\": 0.21718129093129093, \"cost\": 0.007574288250000001, \"time\": 33.07409880757332}, \"ef62e4b003\": {\"quality\": 0.1578144078144078, \"cost\": 0.0220901955, \"time\": 19.725072503089905}, \"efe8e44ba4\": {\"quality\": 0.25713078588078586, \"cost\": 0.00423088875, \"time\": 25.126209509372714}, \"efebd8ca8a\": {\"quality\": 0.12043345543345543, \"cost\": 0.006324273500000001, \"time\": 43.206120282411575}, \"f0a1a27ee9\": {\"quality\": 0.25917291042291035, \"cost\": 0.009095101500000001, \"time\": 33.561097860336304}, \"f1c116068c\": {\"quality\": 0.23332244144744144, \"cost\": 0.00174076875, \"time\": 18.031138664484025}, \"f2c938846b\": {\"quality\": 0.262278971028971, \"cost\": 0.025320526250000003, \"time\": 19.31320353746414}, \"f462950863\": {\"quality\": 0.14478951603951604, \"cost\": 0.0007027954999999999, \"time\": 15.553800004720689}, \"f53c8d7a08\": {\"quality\": 0.23630674880674882, \"cost\": 0.0337485395, \"time\": 42.32889723777771}, \"f570083655\": {\"quality\": 0.21157523032523032, \"cost\": 0.014973397000000003, \"time\": 23.602320194244385}, \"f5f303dab2\": {\"quality\": 0.2371788628038628, \"cost\": 0.032701334, \"time\": 22.13268209695816}, \"f70f78f75a\": {\"quality\": 0.27127698690198687, \"cost\": 0.016446131000000003, \"time\": 12.967699444293977}, \"f767c00a27\": {\"quality\": 0.18858911921411922, \"cost\": 0.007980581, \"time\": 24.888762706518172}, \"f9cf3c3d99\": {\"quality\": 0.23966144966144964, \"cost\": 0.014676935000000002, \"time\": 24.44706709384918}, \"f9e84d96b6\": {\"quality\": 0.234640984015984, \"cost\": 0.0256412165, \"time\": 28.137858784198762}, \"faa5cc0998\": {\"quality\": 0.23488719613719616, \"cost\": 0.0153379235, \"time\": 29.529147928953172}, \"faa744b8b7\": {\"quality\": 0.1879650210900211, \"cost\": 0.008215464999999998, \"time\": 26.75429188609123}, \"fabf1b0f34\": {\"quality\": 0.2536892274392274, \"cost\": 0.0017171129999999997, \"time\": 13.97360804080963}, \"fb53f9a5e6\": {\"quality\": 0.22727897102897102, \"cost\": 0.031424654, \"time\": 43.32508450746536}, \"fe3b87fd92\": {\"quality\": 0.2024367993117993, \"cost\": 0.003581219, \"time\": 22.43371703028679}}" }, { "path": "abacus-research/biodex-revision-priors-mincost.json", "content": "{\"00b02360ef\": {\"quality\": 0.25047369297369293, \"cost\": 0.038505449000000004, \"time\": 33.970101940631864}, \"025a41642e\": {\"quality\": 0.2696782384282384, \"cost\": 0.010557185, \"time\": 31.072653269767763}, \"03c4bc9bb0\": {\"quality\": 0.2581852869352869, \"cost\": 0.010541456000000001, \"time\": 16.739641118049622}, \"04ed954ae9\": {\"quality\": 0.2958458208458209, \"cost\": 0.046920197000000004, \"time\": 19.630667555332185}, \"059dee8af9\": {\"quality\": 0.23122460872460873, \"cost\": 0.008803890499999998, \"time\": 29.826552951335906}, \"05fde83b5e\": {\"quality\": 0.20872460872460877, \"cost\": 0.006782864000000001, \"time\": 30.33404096364975}, \"06b62ff472\": {\"quality\": 0.2968228993228993, \"cost\": 0.005058610000000001, \"time\": 14.343733203411102}, \"0730a1221a\": {\"quality\": 0.26417915417915416, \"cost\": 0.026461371, \"time\": 30.643629491329193}, \"07aec969dd\": {\"quality\": 0.22977744477744477, \"cost\": 0.000723774, \"time\": 38.15098943710327}, \"0a11f0ae2e\": {\"quality\": 0.26241633366633366, \"cost\": 0.009693431000000002, \"time\": 17.0743106007576}, \"0aeb79b24c\": {\"quality\": 0.19304639804639806, \"cost\": 0.0058567, \"time\": 8.773907899856567}, \"0b3eafcb23\": {\"quality\": 0.16931984681984683, \"cost\": 0.0012721839999999998, \"time\": 16.44918472766876}, \"0bc5bbe9f8\": {\"quality\": 0.2640023865023865, \"cost\": 0.030501102000000002, \"time\": 42.8920240521431}, \"0c328ddadf\": {\"quality\": 0.21182165057165053, \"cost\": 0.007213844999999999, \"time\": 25.052031695842743}, \"0db4d3e16d\": {\"quality\": 0.24025238650238653, \"cost\": 0.003130782000000001, \"time\": 18.73849000930786}, \"0df97ab2f2\": {\"quality\": 0.2226345876345876, \"cost\": 0.0073693990000000004, \"time\": 38.507711839675906}, \"0e0162da5c\": {\"quality\": 0.2489013764013764, \"cost\": 0.026232925000000008, \"time\": 28.133001935482024}, \"0e0216352f\": {\"quality\": 0.17297064047064048, \"cost\": 0.008939957, \"time\": 34.948954832553866}, \"0ecd21b93a\": {\"quality\": 0.20069014319014317, \"cost\": 0.025977679000000004, \"time\": 28.50129954814911}, \"0ff2bde030\": {\"quality\": 0.2610857198357198, \"cost\": 0.031667818, \"time\": 16.528453254699706}, \"1059f21a6c\": {\"quality\": 0.2190234765234765, \"cost\": 0.000557161, \"time\": 9.67062486410141}, \"112b665311\": {\"quality\": 0.2551910589410589, \"cost\": 0.004582068, \"time\": 23.5158061504364}, \"11db1dbc4f\": {\"quality\": 0.23640137640137643, \"cost\": 0.003211713, \"time\": 21.993061244487762}, \"13145c10ec\": {\"quality\": 0.3172365134865135, \"cost\": 0.030289498000000005, \"time\": 31.429670691490173}, \"147915874a\": {\"quality\": 0.19364468864468862, \"cost\": 0.00046279649999999997, \"time\": 40.85908321142197}, \"16e1e2aca6\": {\"quality\": 0.219499666999667, \"cost\": 0.005164350000000001, \"time\": 12.553999137878417}, \"1914dfad94\": {\"quality\": 0.24569014319014312, \"cost\": 0.024254144999999998, \"time\": 19.75359899997711}, \"1ab9dcd7d6\": {\"quality\": 0.29846792096792096, \"cost\": 0.001003366, \"time\": 21.74862095117569}, \"1dc749411d\": {\"quality\": 0.20150238650238647, \"cost\": 0.009444109000000001, \"time\": 15.746719026565552}, \"1dd84b0308\": {\"quality\": 0.2539410589410589, \"cost\": 0.026399116, \"time\": 27.934893941879274}, \"1dd9d9bef2\": {\"quality\": 0.2544617882117882, \"cost\": 0.007334834500000001, \"time\": 31.84605222940445}, \"1f0ba92b8b\": {\"quality\": 0.27228105228105226, \"cost\": 0.016661861, \"time\": 33.20399481058121}, \"1f1b5a019d\": {\"quality\": 0.2561663336663337, \"cost\": 0.03240448, \"time\": 22.484670639038086}, \"1f41a35dd3\": {\"quality\": 0.07833333333333332, \"cost\": 0.004150375499999999, \"time\": 15.255917310714722}, \"212c3fccbf\": {\"quality\": 0.3162624875124875, \"cost\": 0.030542124999999996, \"time\": 23.602639937400816}, \"219d51cd03\": {\"quality\": 0.1531672494172494, \"cost\": 0.0008238629999999999, \"time\": 15.20914809703827}, \"238ed27a56\": {\"quality\": 0.28265318015318014, \"cost\": 0.028584483000000004, \"time\": 29.401537466049195}, \"23f3d97d47\": {\"quality\": 0.28292915417915415, \"cost\": 0.026455834000000004, \"time\": 34.8084576010704}, \"253547264d\": {\"quality\": 0.20348651348651345, \"cost\": 0.0261750685, \"time\": 20.272440803050994}, \"27e14a0973\": {\"quality\": 0.2700124875124875, \"cost\": 0.031140450000000004, \"time\": 27.74119987487793}, \"29960739d3\": {\"quality\": 0.18854728604728602, \"cost\": 0.030525630000000005, \"time\": 26.206378316879274}, \"2b0a8d5614\": {\"quality\": 0.2262060162060162, \"cost\": 0.021956766, \"time\": 16.901934444904327}, \"2ccd104a68\": {\"quality\": 0.23872460872460874, \"cost\": 0.017867876999999997, \"time\": 46.65757336616516}, \"2d1b0cd5d9\": {\"quality\": 0.2476134976134976, \"cost\": 0.005795422000000001, \"time\": 17.555127108097075}, \"2dcffa3d30\": {\"quality\": 0.3144208569208569, \"cost\": 0.008584119999999999, \"time\": 24.954598808288573}, \"2ebe033e6a\": {\"quality\": 0.2554706404706405, \"cost\": 0.007171497000000001, \"time\": 16.872591602802277}, \"31e8ef9410\": {\"quality\": 0.06845238095238096, \"cost\": 0.0007121749999999999, \"time\": 75.84835036993027}, \"32a628c971\": {\"quality\": 0.31888333888333886, \"cost\": 0.035293712000000005, \"time\": 27.239106237888336}, \"336e1a4fdc\": {\"quality\": 0.22028138528138527, \"cost\": 0.030691956000000003, \"time\": 36.1869512796402}, \"33e0fdf4e1\": {\"quality\": 0.2365234765234765, \"cost\": 0.005039102, \"time\": 23.91675148010254}, \"38a60c768a\": {\"quality\": 0.2449529637029637, \"cost\": 0.025963075000000002, \"time\": 34.96920952796936}, \"39bb54fbfb\": {\"quality\": 0.29789127539127536, \"cost\": 0.030766864999999997, \"time\": 45.52991482019424}, \"3ace395562\": {\"quality\": 0.24872460872460872, \"cost\": 0.024912788, \"time\": 23.518352210521698}, \"3bbe08bf63\": {\"quality\": 0.33796703296703295, \"cost\": 0.032394053000000006, \"time\": 37.81355751752854}, \"3caf8d77c5\": {\"quality\": 0.3083079420579421, \"cost\": 0.03448324800000001, \"time\": 29.443913543224333}, \"3d25abc0cb\": {\"quality\": 0.2672743922743922, \"cost\": 0.008724563000000001, \"time\": 37.292464637756346}, \"3e19290ef5\": {\"quality\": 0.29275238650238655, \"cost\": 0.024963852000000005, \"time\": 22.140427541732787}, \"3e8e089a02\": {\"quality\": 0.280807942057942, \"cost\": 0.0233879175, \"time\": 11.965235376358033}, \"3f1771caee\": {\"quality\": 0.143228576978577, \"cost\": 0.003751839, \"time\": 24.353824079036713}, \"3f7beb53c4\": {\"quality\": 0.29019411144411145, \"cost\": 0.00168912, \"time\": 15.684711122512818}, \"4081f36f1d\": {\"quality\": 0.2203912753912754, \"cost\": 0.008877949000000001, \"time\": 35.81259698867798}, \"410edf5f69\": {\"quality\": 0.2731690531690532, \"cost\": 0.032445931, \"time\": 40.782276201248166}, \"44ec78f301\": {\"quality\": 0.2575774225774226, \"cost\": 0.030435683499999998, \"time\": 29.96491810083389}, \"44eeb11408\": {\"quality\": 0.27455794205794204, \"cost\": 0.026113869, \"time\": 22.3088561296463}, \"45623e6def\": {\"quality\": 0.34303016428016425, \"cost\": 0.032769938, \"time\": 30.375658011436464}, \"470749179f\": {\"quality\": 0.26199966699966704, \"cost\": 0.004939885999999999, \"time\": 24.02986795902252}, \"47ea56eba0\": {\"quality\": 0.2065848040848041, \"cost\": 0.017984692000000004, \"time\": 29.315977609157564}, \"49fc8b3768\": {\"quality\": 0.16132201132201132, \"cost\": 0.002801354, \"time\": 18.934259843826293}, \"4a60adbf47\": {\"quality\": 0.17342823842823843, \"cost\": 0.013229327499999999, \"time\": 26.285319995880126}, \"4ba372d292\": {\"quality\": 0.26265623265623267, \"cost\": 0.0073457165, \"time\": 27.0092391371727}, \"4bf38b1922\": {\"quality\": 0.24983571983571987, \"cost\": 0.030574272000000006, \"time\": 43.60012021064758}, \"4c415b4b86\": {\"quality\": 0.15533175158175158, \"cost\": 0.024658497000000005, \"time\": 14.953267228603362}, \"4d1fc31b49\": {\"quality\": 0.3281690531690532, \"cost\": 0.023400054000000003, \"time\": 16.318808138370514}, \"4d8c84f53a\": {\"quality\": 0.2701134976134976, \"cost\": 0.026254961000000004, \"time\": 29.027335906028746}, \"4df2245b9b\": {\"quality\": 0.12687756687756688, \"cost\": 0.001181844, \"time\": 4.539927685260773}, \"4e9274d39b\": {\"quality\": 0.2406522644022644, \"cost\": 0.005825131000000001, \"time\": 15.826024615764618}, \"4f87ed7c2e\": {\"quality\": 0.2662246087246087, \"cost\": 0.008736922999999999, \"time\": 27.13904390335083}, \"51a340bebc\": {\"quality\": 0.24584582084582088, \"cost\": 0.0313118765, \"time\": 28.96012043952942}, \"52dee74e25\": {\"quality\": 0.1881456043956044, \"cost\": 0.031453754, \"time\": 34.28346945047379}, \"533fddb3bd\": {\"quality\": 0.13842823842823843, \"cost\": 0.00165954, \"time\": 9.557414150238037}, \"5358cb5855\": {\"quality\": 0.18078934953934955, \"cost\": 0.026390755999999998, \"time\": 21.654051101207735}, \"53b6c3e00f\": {\"quality\": 0.25569014319014316, \"cost\": 0.000717601, \"time\": 10.53179224729538}, \"5400c14640\": {\"quality\": 0.30376248751248747, \"cost\": 0.028650208999999992, \"time\": 38.553849005699156}, \"55e373a7e2\": {\"quality\": 0.12201770451770451, \"cost\": 0.039114879000000005, \"time\": 18.304805862903596}, \"5652070ecb\": {\"quality\": 0.21717698967698967, \"cost\": 0.007322918, \"time\": 32.560013604164126}, \"56c5cbc75a\": {\"quality\": 0.21707903207903206, \"cost\": 0.000547569, \"time\": 38.314506363868716}, \"577b168d21\": {\"quality\": 0.22071178821178825, \"cost\": 0.006765484, \"time\": 12.956210255622864}, \"57cabf0e4b\": {\"quality\": 0.21348651348651346, \"cost\": 0.024673236, \"time\": 20.83515272140503}, \"58dbfc0499\": {\"quality\": 0.3025124875124875, \"cost\": 0.053043769000000004, \"time\": 33.01721404790878}, \"5a5739c3d2\": {\"quality\": 0.16318528693528694, \"cost\": 0.006033024, \"time\": 30.727768671512603}, \"5b12979e52\": {\"quality\": 0.15083694083694082, \"cost\": 0.0036988935, \"time\": 18.811239528656007}, \"5b384da665\": {\"quality\": 0.24699841824841826, \"cost\": 0.014450417999999998, \"time\": 25.65202077627182}, \"5b837baf2e\": {\"quality\": 0.2600124875124875, \"cost\": 0.03879646400000001, \"time\": 38.47186998128891}, \"5c8acd9a75\": {\"quality\": 0.2188238150738151, \"cost\": 0.026283571999999998, \"time\": 32.48901460170746}, \"5da68874af\": {\"quality\": 0.14364649239649238, \"cost\": 0.008979280499999999, \"time\": 33.84627612829208}, \"5e304846b6\": {\"quality\": 0.11458083583083582, \"cost\": 0.007788067, \"time\": 22.096365022659302}, \"5e608d2a8b\": {\"quality\": 0.15110375735375733, \"cost\": 0.0008234890000000002, \"time\": 19.88741739988327}, \"5e6ad9fb8a\": {\"quality\": 0.24834582084582085, \"cost\": 0.024505677000000003, \"time\": 15.4741956949234}, \"5ec5b185c9\": {\"quality\": 0.2114807414807415, \"cost\": 0.0023043549999999997, \"time\": 37.4191596031189}, \"608e332141\": {\"quality\": 0.34233571983571986, \"cost\": 0.035189383000000005, \"time\": 22.62248021364212}, \"610753ffe5\": {\"quality\": 0.31424048174048175, \"cost\": 0.029135328, \"time\": 19.57549432516098}, \"614e59c7e7\": {\"quality\": 0.2492218892218892, \"cost\": 0.001694424, \"time\": 18.55128355026245}, \"62320710f0\": {\"quality\": 0.3199746087246087, \"cost\": 0.03502582600000001, \"time\": 31.618607234954833}, \"649df27d73\": {\"quality\": 0.2782466144966145, \"cost\": 0.032545105, \"time\": 19.415873074531554}, \"64f956ba72\": {\"quality\": 0.2931690531690532, \"cost\": 0.026772999999999998, \"time\": 9.635773408412934}, \"65657a3d04\": {\"quality\": 0.20616633366633366, \"cost\": 0.011125794, \"time\": 52.96317781209946}, \"6604abbd43\": {\"quality\": 0.30366633366633367, \"cost\": 0.025960788500000005, \"time\": 24.2287339925766}, \"67050fa89b\": {\"quality\": 0.25265623265623266, \"cost\": 0.030506207, \"time\": 39.747187566757205}, \"67f05c5985\": {\"quality\": 0.24658300033300035, \"cost\": 0.02588163, \"time\": 20.159594106674195}, \"67fe27bce0\": {\"quality\": 0.3328912753912754, \"cost\": 0.039865689, \"time\": 32.06505537033081}, \"698d1b1d0e\": {\"quality\": 0.31830794205794205, \"cost\": 0.03946355800000001, \"time\": 51.54699250459671}, \"69e8aa9f73\": {\"quality\": 0.18985680985680986, \"cost\": 0.007278381, \"time\": 38.63639385700226}, \"6b240a8971\": {\"quality\": 0.20597027972027973, \"cost\": 0.008727093000000002, \"time\": 28.28091263771057}, \"6b56430005\": {\"quality\": 0.3260460372960373, \"cost\": 0.026531200000000005, \"time\": 41.742786502838136}, \"6b611b7193\": {\"quality\": 0.2922743922743923, \"cost\": 0.066505375, \"time\": 17.20961241722107}, \"6b77ef93d3\": {\"quality\": 0.17878232878232875, \"cost\": 0.001498626, \"time\": 14.678290736675262}, \"6beefce17b\": {\"quality\": 0.30852439227439227, \"cost\": 0.030691372000000005, \"time\": 24.38176097869873}, \"6cb7c9a802\": {\"quality\": 0.33327006327006325, \"cost\": 0.030604636500000004, \"time\": 39.88295543193817}, \"6cba1da5b3\": {\"quality\": 0.15214035964035963, \"cost\": 0.013090929000000001, \"time\": 21.816087329387663}, \"6dea54bf99\": {\"quality\": 0.2879309579309579, \"cost\": 0.004796466000000001, \"time\": 23.340252709388732}, \"6eda6f7780\": {\"quality\": 0.2093607781107781, \"cost\": 0.010549649000000001, \"time\": 36.39340627193451}, \"708f3ca5da\": {\"quality\": 0.13352439227439225, \"cost\": 0.0014636204999999998, \"time\": 14.394513404369354}, \"72d75a5f01\": {\"quality\": 0.22862359862359863, \"cost\": 0.014409670999999999, \"time\": 24.5260990858078}, \"72d99dee93\": {\"quality\": 0.2751513764013764, \"cost\": 0.028088187000000008, \"time\": 41.42688899040222}, \"73272e5bd3\": {\"quality\": 0.3387246087246087, \"cost\": 0.030827000000000004, \"time\": 27.08615951538086}, \"74608570f5\": {\"quality\": 0.11727564102564103, \"cost\": 0.000531192, \"time\": 25.665570771694185}, \"7531840182\": {\"quality\": 0.1608241758241758, \"cost\": 0.0018236030000000005, \"time\": 65.24554077386856}, \"77ba656673\": {\"quality\": 0.15628538128538128, \"cost\": 0.008854519500000001, \"time\": 28.12923115491867}, \"77fe41f02f\": {\"quality\": 0.15589035964035963, \"cost\": 0.042701628000000005, \"time\": 38.7138086438179}, \"7857e95756\": {\"quality\": 0.20908480408480407, \"cost\": 0.014196858000000001, \"time\": 26.021716463565827}, \"78984e94f0\": {\"quality\": 0.19880702630702632, \"cost\": 0.0067921480000000005, \"time\": 24.860538172721864}, \"7986633543\": {\"quality\": 0.25714035964035964, \"cost\": 0.0073338084999999996, \"time\": 35.33250515460968}, \"7bdbc32b57\": {\"quality\": 0.3116196303696303, \"cost\": 0.0451492925, \"time\": 26.672986245155336}, \"7c89cff787\": {\"quality\": 0.1501694139194139, \"cost\": 0.0016053209999999998, \"time\": 20.911856007575988}, \"7dc51ec191\": {\"quality\": 0.34112359862359865, \"cost\": 0.039412432000000004, \"time\": 45.49512637853623}, \"7e4a6245ac\": {\"quality\": 0.13822802197802198, \"cost\": 0.0004819685, \"time\": 76.24876043796539}, \"7ebc55ebcf\": {\"quality\": 0.3203912753912754, \"cost\": 0.008825814999999997, \"time\": 52.15610808134079}, \"7f14114a48\": {\"quality\": 0.21598651348651346, \"cost\": 0.012735542499999999, \"time\": 17.204772758483887}, \"7fb587503e\": {\"quality\": 0.22814560439560436, \"cost\": 0.009006149000000001, \"time\": 18.705357003211976}, \"7ff82b6f8d\": {\"quality\": 0.22926948051948054, \"cost\": 0.028896988000000002, \"time\": 20.79049743413925}, \"806a5ef096\": {\"quality\": 0.27122460872460874, \"cost\": 0.023206115, \"time\": 11.138013732433318}, \"8156d78e42\": {\"quality\": 0.251999666999667, \"cost\": 0.0073474729999999985, \"time\": 36.957821094989775}, \"815cfb848c\": {\"quality\": 0.2520958208458209, \"cost\": 0.0247796475, \"time\": 23.402184796333312}, \"820b42e0b1\": {\"quality\": 0.2400949050949051, \"cost\": 0.002655385, \"time\": 22.863080990314483}, \"82a733626e\": {\"quality\": 0.30914127539127534, \"cost\": 0.028937054, \"time\": 23.70889393091202}, \"83d6300ced\": {\"quality\": 0.2575124875124875, \"cost\": 0.008893523, \"time\": 39.3106644153595}, \"844c822d56\": {\"quality\": 0.15672369297369299, \"cost\": 0.02642007, \"time\": 22.442726397514342}, \"84f9f20f75\": {\"quality\": 0.2695976245976246, \"cost\": 0.023199555000000004, \"time\": 15.404870545864105}, \"8525986a00\": {\"quality\": 0.2975124875124875, \"cost\": 0.030195346000000005, \"time\": 38.38539198637009}, \"85db012139\": {\"quality\": 0.2975142912642913, \"cost\": 0.04670653, \"time\": 23.935909843444826}, \"85ddc8a71e\": {\"quality\": 0.2362246087246087, \"cost\": 0.005867745, \"time\": 22.264061617851258}, \"8767267bed\": {\"quality\": 0.1742567155067155, \"cost\": 0.006601124, \"time\": 7.172838580608368}, \"882238f677\": {\"quality\": 0.14891941391941393, \"cost\": 0.003255827, \"time\": 27.542255055904388}, \"88f78bdeaf\": {\"quality\": 0.09306623931623932, \"cost\": 0.000593519, \"time\": 27.451544547080992}, \"8af1605300\": {\"quality\": 0.29047369297369297, \"cost\": 0.007263364000000003, \"time\": 27.19462056159973}, \"8c09cba5e2\": {\"quality\": 0.24566905316905316, \"cost\": 0.0249850575, \"time\": 30.331667792797088}, \"8e5d966cf6\": {\"quality\": 0.13822802197802198, \"cost\": 0.001409651, \"time\": 68.11073198318482}, \"8eacbc7240\": {\"quality\": 0.21547369297369298, \"cost\": 0.012992563000000002, \"time\": 23.104503071308137}, \"8fc917c575\": {\"quality\": 0.24955794205794207, \"cost\": 0.007428123, \"time\": 26.517943835258485}, \"8fe43ec148\": {\"quality\": 0.25275724275724276, \"cost\": 0.0030687225, \"time\": 27.1856760263443}, \"900de152da\": {\"quality\": 0.28334582084582083, \"cost\": 0.026874375, \"time\": 8.219515359401703}, \"901ab65b55\": {\"quality\": 0.34566905316905316, \"cost\": 0.027035125000000004, \"time\": 9.139500224590302}, \"918182aa18\": {\"quality\": 0.18560592185592187, \"cost\": 0.002369203, \"time\": 56.10349173545838}, \"91ac37a21a\": {\"quality\": 0.128228021978022, \"cost\": 0.006092536000000001, \"time\": 74.77904181480407}, \"9252afa4ec\": {\"quality\": 0.2821373071373071, \"cost\": 0.024976994000000002, \"time\": 19.884121739864348}, \"939b0c3ed9\": {\"quality\": 0.24926157176157174, \"cost\": 0.009163022, \"time\": 20.210047280788423}, \"93a705cb0d\": {\"quality\": 0.25301157176157174, \"cost\": 0.01034815, \"time\": 28.735098361968994}, \"94091e2968\": {\"quality\": 0.20182109557109557, \"cost\": 0.010538336, \"time\": 30.044746005535124}, \"945a9565fa\": {\"quality\": 0.177733793983794, \"cost\": 0.005453404, \"time\": 10.252689445018769}, \"946baa620a\": {\"quality\": 0.2701134976134976, \"cost\": 0.024427862, \"time\": 11.070768821239472}, \"950ed976dd\": {\"quality\": 0.09488455988455988, \"cost\": 0.0034412205, \"time\": 20.066544604301452}, \"955689ee8f\": {\"quality\": 0.17957417582417584, \"cost\": 0.0017661469999999998, \"time\": 37.72812922000885}, \"95c9b1b1dd\": {\"quality\": 0.2563003663003663, \"cost\": 0.030597902, \"time\": 27.93373385667801}, \"95ebba630d\": {\"quality\": 0.19334887334887335, \"cost\": 0.004341687, \"time\": 18.660895478725433}, \"966282f059\": {\"quality\": 0.16358821733821732, \"cost\": 0.022551696000000003, \"time\": 21.573746180534364}, \"96c401772b\": {\"quality\": 0.2422365134865135, \"cost\": 0.0026645590000000004, \"time\": 17.938199400901794}, \"9718d880f6\": {\"quality\": 0.20941905316905318, \"cost\": 0.030509630000000003, \"time\": 42.891600167751314}, \"971ae914cb\": {\"quality\": 0.3079922854922855, \"cost\": 0.024098346, \"time\": 11.75359138250351}, \"97566e70dd\": {\"quality\": 0.21477439227439227, \"cost\": 0.016637812999999998, \"time\": 31.738031923770905}, \"97c882ca1f\": {\"quality\": 0.2701134976134976, \"cost\": 0.034874281, \"time\": 22.407317924499512}, \"98e28fbb93\": {\"quality\": 0.18297369297369298, \"cost\": 0.009208929000000001, \"time\": 11.046243464946746}, \"992ab25916\": {\"quality\": 0.24106893106893104, \"cost\": 0.001710615, \"time\": 18.1801389336586}, \"9a271dff29\": {\"quality\": 0.20304639804639804, \"cost\": 0.009020588999999999, \"time\": 25.38304090499878}, \"9aa6fd8b03\": {\"quality\": 0.2443198468198468, \"cost\": 0.0058449355000000005, \"time\": 14.622885823249817}, \"9b12b7d441\": {\"quality\": 0.3142615717615717, \"cost\": 0.010453449, \"time\": 24.59365568161011}, \"9bf31a2127\": {\"quality\": 0.2737246087246087, \"cost\": 0.032263425, \"time\": 41.504128229618075}, \"9c4ad5aed3\": {\"quality\": 0.2762624875124875, \"cost\": 0.024755695000000005, \"time\": 21.714357328414916}, \"9cbb34ee5f\": {\"quality\": 0.23064740814740814, \"cost\": 0.0023012260000000004, \"time\": 23.29413582086563}, \"9e16d7ba9c\": {\"quality\": 0.25935772560772563, \"cost\": 0.031369863000000005, \"time\": 33.015732765197754}, \"9e1a7dd196\": {\"quality\": 0.19444194694194694, \"cost\": 0.0035900009999999998, \"time\": 15.451323175430298}, \"9e6775ac33\": {\"quality\": 0.06856227106227106, \"cost\": 0.0017025660000000002, \"time\": 15.161614441871643}, \"9e8674b6c7\": {\"quality\": 0.25872460872460873, \"cost\": 0.032513841, \"time\": 38.995942330360414}, \"9ea12d8d80\": {\"quality\": 0.14647893772893772, \"cost\": 0.006858684000000001, \"time\": 12.53976230621338}, \"a232f1dfb6\": {\"quality\": 0.2482117882117882, \"cost\": 0.0017046329999999999, \"time\": 17.978844976425172}, \"a3b7cb6c33\": {\"quality\": 0.24213369963369963, \"cost\": 0.00900874, \"time\": 19.766272163391115}, \"a72f24f687\": {\"quality\": 0.21526862026862026, \"cost\": 0.007625456000000003, \"time\": 14.816489315032959}, \"a8aac20fd9\": {\"quality\": 0.33620115995115996, \"cost\": 0.027009404000000004, \"time\": 20.422273874282837}, \"a9e811d4a7\": {\"quality\": 0.27455794205794204, \"cost\": 0.023571902, \"time\": 17.510642659664153}, \"aa1fc47a86\": {\"quality\": 0.23011349761349761, \"cost\": 0.024848436999999998, \"time\": 23.83184322118759}, \"aa67b102e7\": {\"quality\": 0.29247460872460873, \"cost\": 0.029215488, \"time\": 25.6765993475914}, \"ad453a813f\": {\"quality\": 0.2581474081474081, \"cost\": 0.0031871219999999997, \"time\": 17.01520048379898}, \"add25d67a0\": {\"quality\": 0.19999750249750248, \"cost\": 0.027308536, \"time\": 21.267851161956788}, \"adf6ae1ba7\": {\"quality\": 0.15185772560772562, \"cost\": 0.0030511870000000003, \"time\": 10.442331075668335}, \"af57afe626\": {\"quality\": 0.25730269730269734, \"cost\": 0.0027442350000000003, \"time\": 20.559476172924043}, \"b07ae35700\": {\"quality\": 0.25820693195693195, \"cost\": 0.0031182555, \"time\": 21.726053476333618}, \"b09741c8f7\": {\"quality\": 0.13642246642246642, \"cost\": 0.00039643100000000004, \"time\": 36.347797584533694}, \"b1b81c0847\": {\"quality\": 0.35997460872460874, \"cost\": 0.0234363775, \"time\": 12.63291654586792}, \"b38fbeda99\": {\"quality\": 0.18547369297369298, \"cost\": 0.008016031, \"time\": 29.251738607883453}, \"b3b92b1835\": {\"quality\": 0.24926157176157174, \"cost\": 0.04590919, \"time\": 22.62844548225403}, \"b3bba3eee2\": {\"quality\": 0.19231532356532358, \"cost\": 0.007909536, \"time\": 42.60630009174347}, \"b40727bd68\": {\"quality\": 0.2634468309468309, \"cost\": 0.024455202000000006, \"time\": 20.662753903865813}, \"b6a0f78896\": {\"quality\": 0.264456931956932, \"cost\": 0.008735882999999998, \"time\": 34.55237965583801}, \"b84e122880\": {\"quality\": 0.2734468309468309, \"cost\": 0.008659572, \"time\": 40.34617894887924}, \"b8c6c44a39\": {\"quality\": 0.30366633366633367, \"cost\": 0.0289365195, \"time\": 22.390970599651336}, \"bb9ed9dffe\": {\"quality\": 0.16283660783660783, \"cost\": 0.001536843, \"time\": 17.358729672431945}, \"bbc50a0411\": {\"quality\": 0.22165931290931287, \"cost\": 0.0031182930000000003, \"time\": 28.581140315532686}, \"bbd0498616\": {\"quality\": 0.22491633366633365, \"cost\": 0.051756142000000005, \"time\": 17.691377997398376}, \"bd018ec9d7\": {\"quality\": 0.19921703296703297, \"cost\": 0.0005695385, \"time\": 72.71849246025086}, \"be11a8a86e\": {\"quality\": 0.2582117882117882, \"cost\": 0.013947379000000001, \"time\": 26.179900193214415}, \"bfa93a259c\": {\"quality\": 0.3037246087246087, \"cost\": 0.0248712575, \"time\": 27.526713395118712}, \"c127942d8c\": {\"quality\": 0.2636650849150849, \"cost\": 0.0032525069999999995, \"time\": 24.242635214328764}, \"c14216d744\": {\"quality\": 0.24926157176157174, \"cost\": 0.010124539499999998, \"time\": 25.00144135951996}, \"c2f0bc6921\": {\"quality\": 0.14888458763458762, \"cost\": 0.000605392, \"time\": 14.55265941619873}, \"c440b67f31\": {\"quality\": 0.21598651348651346, \"cost\": 0.02981187, \"time\": 24.81428416967392}, \"caa29fbe8c\": {\"quality\": 0.27789127539127534, \"cost\": 0.0247254, \"time\": 26.996189880371094}, \"cd46331132\": {\"quality\": 0.30287448662448657, \"cost\": 0.005876232, \"time\": 16.816848158836365}, \"cdd01242f5\": {\"quality\": 0.3245579420579421, \"cost\": 0.029167213999999997, \"time\": 18.141655015945435}, \"cdf27f19d3\": {\"quality\": 0.2780494505494505, \"cost\": 0.005842002000000001, \"time\": 15.971444475650788}, \"ce7d236454\": {\"quality\": 0.28977258852258847, \"cost\": 0.030690639, \"time\": 30.128969633579253}, \"d117c8e23b\": {\"quality\": 0.3412246087246087, \"cost\": 0.03989662200000001, \"time\": 39.21010599136353}, \"d27a5ad38a\": {\"quality\": 0.1940848040848041, \"cost\": 0.024636320000000003, \"time\": 19.60495171546936}, \"d287ceaae7\": {\"quality\": 0.3262624875124875, \"cost\": 0.035333185, \"time\": 32.71745250225067}, \"d29027747a\": {\"quality\": 0.0815018315018315, \"cost\": 0.000420386, \"time\": 37.273941445350644}, \"d2d7780d31\": {\"quality\": 0.2843198468198468, \"cost\": 0.031180686000000006, \"time\": 28.442743134498595}, \"d566a94f3a\": {\"quality\": 0.30195873570873577, \"cost\": 0.02176175, \"time\": 8.928415286540986}, \"d64dd29abb\": {\"quality\": 0.3114410589410589, \"cost\": 0.03748003400000001, \"time\": 31.35522733926773}, \"d75495aed3\": {\"quality\": 0.24872460872460872, \"cost\": 0.017243897, \"time\": 37.48555475473404}, \"d93337a87d\": {\"quality\": 0.2603912753912754, \"cost\": 0.009152621000000001, \"time\": 21.659664070606233}, \"de08b84b51\": {\"quality\": 0.2466244866244866, \"cost\": 0.005220604999999999, \"time\": 13.204281628131866}, \"de39f4f113\": {\"quality\": 0.1780698468198468, \"cost\": 0.045369664999999997, \"time\": 15.614853060245514}, \"de83615ac5\": {\"quality\": 0.32421883671883667, \"cost\": 0.0074163139999999985, \"time\": 25.368940019607543}, \"dfa1e5267d\": {\"quality\": 0.24926157176157174, \"cost\": 0.001261335, \"time\": 7.206225287914276}, \"dfedfba6df\": {\"quality\": 0.24569014319014312, \"cost\": 0.0042948735, \"time\": 20.24753314256668}, \"e15d59fc9d\": {\"quality\": 0.33080794205794206, \"cost\": 0.011172813, \"time\": 53.19109081029892}, \"e540f66a9d\": {\"quality\": 0.14333083583083583, \"cost\": 0.024458602000000006, \"time\": 16.43147575855255}, \"e5c7c9fce2\": {\"quality\": 0.28936258186258185, \"cost\": 0.024852987500000003, \"time\": 23.036948883533476}, \"e73beb7df9\": {\"quality\": 0.1644678932178932, \"cost\": 0.0206705125, \"time\": 19.65618509054184}, \"e939aaa8b3\": {\"quality\": 0.24094683094683092, \"cost\": 0.026475653000000005, \"time\": 27.854652655124664}, \"e9e55ba5db\": {\"quality\": 0.16214035964035964, \"cost\": 0.024552966000000002, \"time\": 22.01356840133667}, \"ea817a074f\": {\"quality\": 0.2803912753912754, \"cost\": 0.030159274000000003, \"time\": 26.384180903434753}, \"eca3d11e40\": {\"quality\": 0.25997460872460876, \"cost\": 0.024926519, \"time\": 22.995232653617858}, \"ed945b7e4a\": {\"quality\": 0.28362540237540235, \"cost\": 0.030488778000000005, \"time\": 23.277513599395753}, \"ee4ddebe70\": {\"quality\": 0.2726959151959152, \"cost\": 0.007441470000000001, \"time\": 33.65448969602585}, \"ef62e4b003\": {\"quality\": 0.15812881562881562, \"cost\": 0.020466967000000003, \"time\": 19.30031136274338}, \"efe8e44ba4\": {\"quality\": 0.2592615717615717, \"cost\": 0.0040250775, \"time\": 21.48128048181534}, \"efebd8ca8a\": {\"quality\": 0.11086691086691088, \"cost\": 0.004719911, \"time\": 28.901436603069307}, \"f0a1a27ee9\": {\"quality\": 0.25251248751248745, \"cost\": 0.008780909, \"time\": 30.412869799137116}, \"f1c116068c\": {\"quality\": 0.2574782162282162, \"cost\": 0.001708935, \"time\": 17.96944682598114}, \"f2c938846b\": {\"quality\": 0.32955794205794203, \"cost\": 0.024549196500000002, \"time\": 19.827334237098693}, \"f462950863\": {\"quality\": 0.18207903207903206, \"cost\": 0.0006345659999999999, \"time\": 19.180297017097473}, \"f53c8d7a08\": {\"quality\": 0.27761349761349763, \"cost\": 0.032353539, \"time\": 41.01129903793335}, \"f570083655\": {\"quality\": 0.25315046065046065, \"cost\": 0.014203856, \"time\": 25.355476438999176}, \"f5f303dab2\": {\"quality\": 0.26935772560772564, \"cost\": 0.031210957999999994, \"time\": 26.234438502788542}, \"f70f78f75a\": {\"quality\": 0.31005397380397376, \"cost\": 0.015462705999999998, \"time\": 15.054353976249695}, \"f767c00a27\": {\"quality\": 0.1513449050949051, \"cost\": 0.007778000499999998, \"time\": 23.562188839912416}, \"f9cf3c3d99\": {\"quality\": 0.30015623265623265, \"cost\": 0.013173157, \"time\": 23.90029693841934}, \"f9e84d96b6\": {\"quality\": 0.2451153013653013, \"cost\": 0.024730744000000002, \"time\": 21.731143033504488}, \"faa5cc0998\": {\"quality\": 0.27644105894105897, \"cost\": 0.014485047000000001, \"time\": 30.353227543830872}, \"faa744b8b7\": {\"quality\": 0.17509670884670886, \"cost\": 0.007972195999999999, \"time\": 26.635566544532775}, \"fabf1b0f34\": {\"quality\": 0.3082117882117882, \"cost\": 0.0016653059999999996, \"time\": 13.257695925235748}, \"fb53f9a5e6\": {\"quality\": 0.312057942057942, \"cost\": 0.030517666500000006, \"time\": 38.16447761058807}, \"fe3b87fd92\": {\"quality\": 0.2373735986235986, \"cost\": 0.003384599, \"time\": 24.418578135967255}}" }, { "path": "abacus-research/cheap-priors-cascades.json", "content": "{\"0005c18b69\": {\"quality\": 0.5422666666666667, \"cost\": 1.62e-06, \"time\": 0.026}, \"009df798a3\": {\"quality\": 0.470225, \"cost\": 1.59e-06, \"time\": 0.027999999999999997}, \"00c93aec22\": {\"quality\": 0.426725, \"cost\": 3.1199999999999998e-06, \"time\": 0.030900000000000004}, \"00e1fecc4c\": {\"quality\": 0.48889999999999995, \"cost\": 2.24e-06, \"time\": 0.026199999999999998}, \"00f4acd0d3\": {\"quality\": 0.49876666666666675, \"cost\": 3.15e-06, \"time\": 0.0322}, \"01413aa72d\": {\"quality\": 0.44705, \"cost\": 2.3600000000000003e-06, \"time\": 0.0293}, \"01c2f973ad\": {\"quality\": 0.5318, \"cost\": 1.95e-06, \"time\": 0.020999999999999998}, \"02078988c1\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.031}, \"021604dec1\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"02410c662e\": {\"quality\": 0.4013, \"cost\": 1.8e-06, \"time\": 0.0299}, \"0262668df7\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"0267c97b70\": {\"quality\": 0.5648333333333334, \"cost\": 2.7e-06, \"time\": 0.0308}, \"02ae38e4aa\": {\"quality\": 0.45935000000000004, \"cost\": 1.59e-06, \"time\": 0.0247}, \"02f49fe0fd\": {\"quality\": 0.365, \"cost\": 2.4e-07, \"time\": 0.0128}, \"030756558c\": {\"quality\": 0.45363333333333333, \"cost\": 9.9e-07, \"time\": 0.0226}, \"033ca325e6\": {\"quality\": 0.5508500000000001, \"cost\": 2.12e-06, \"time\": 0.019799999999999998}, \"038a5f0a62\": {\"quality\": 0.5648333333333334, \"cost\": 2.7e-06, \"time\": 0.030799999999999998}, \"041b5af43d\": {\"quality\": 0.52195, \"cost\": 3.830000000000001e-06, \"time\": 0.0413}, \"042d933706\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"04397effa0\": {\"quality\": 0.46777500000000005, \"cost\": 2.43e-06, \"time\": 0.0333}, \"0539e0b42d\": {\"quality\": 0.5050250000000001, \"cost\": 2.7500000000000004e-06, \"time\": 0.0365}, \"0554568b86\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.0229}, \"06493715cc\": {\"quality\": 0.4763, \"cost\": 1.47e-06, \"time\": 0.015}, \"067ee6e91b\": {\"quality\": 0.49876666666666675, \"cost\": 3.15e-06, \"time\": 0.0322}, \"068b66f00d\": {\"quality\": 0.4630666666666667, \"cost\": 2e-06, \"time\": 0.026699999999999998}, \"0695f9b5fc\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"073ed5b301\": {\"quality\": 0.39890000000000003, \"cost\": 1.6799999999999998e-06, \"time\": 0.020200000000000003}, \"079feb14a8\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.0263}, \"07a3a7daf7\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"08127cd6dd\": {\"quality\": 0.6497666666666667, \"cost\": 4.27e-06, \"time\": 0.037599999999999995}, \"0833133620\": {\"quality\": 0.5536, \"cost\": 1.86e-06, \"time\": 0.022199999999999998}, \"089565077c\": {\"quality\": 0.365, \"cost\": 1.2e-07, \"time\": 0.0064}, \"08bf8cc191\": {\"quality\": 0.53045, \"cost\": 4.07e-06, \"time\": 0.0375}, \"08e1802287\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"090cd3ef31\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"0944a921e8\": {\"quality\": 0.48890000000000006, \"cost\": 2.24e-06, \"time\": 0.026199999999999998}, \"0947216ece\": {\"quality\": 0.5724666666666667, \"cost\": 3.88e-06, \"time\": 0.0304}, \"096d51f670\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"09791c731b\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"0990c0d4f8\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"0a128688c1\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.0229}, \"0a4c1bbb4a\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.018299999999999997}, \"0ac969dde3\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.040799999999999996}, \"0af1efab0e\": {\"quality\": 0.4021666666666666, \"cost\": 8.4e-07, \"time\": 0.0149}, \"0b1ed7ff58\": {\"quality\": 0.5244, \"cost\": 2.99e-06, \"time\": 0.036}, \"0b3dc2e896\": {\"quality\": 0.45555, \"cost\": 2.6e-06, \"time\": 0.025500000000000002}, \"0b43e94f3f\": {\"quality\": 0.5121, \"cost\": 1.88e-06, \"time\": 0.0203}, \"0b4ab72197\": {\"quality\": 0.42146666666666666, \"cost\": 2.76e-06, \"time\": 0.025}, \"0be862a0dc\": {\"quality\": 0.4762, \"cost\": 2.07e-06, \"time\": 0.027399999999999997}, \"0bf3129ae8\": {\"quality\": 0.4794666666666667, \"cost\": 1.23e-06, \"time\": 0.0221}, \"0c020b86a3\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.0224}, \"0c6c7fe96a\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"0c81c8996a\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"0cdc5954dd\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.031}, \"0d25188bf7\": {\"quality\": 0.4098, \"cost\": 2.04e-06, \"time\": 0.0261}, \"0d9d767ae5\": {\"quality\": 0.5611333333333334, \"cost\": 3.6400000000000003e-06, \"time\": 0.0342}, \"0e36342fe7\": {\"quality\": 0.39885000000000004, \"cost\": 1.3199999999999999e-06, \"time\": 0.0176}, \"0e7e862290\": {\"quality\": 0.4021666666666666, \"cost\": 8.4e-07, \"time\": 0.0149}, \"0e91cd07f9\": {\"quality\": 0.4762, \"cost\": 2.07e-06, \"time\": 0.027399999999999997}, \"0ec672e7c8\": {\"quality\": 0.5304500000000001, \"cost\": 4.07e-06, \"time\": 0.037500000000000006}, \"0ed243f788\": {\"quality\": 0.543775, \"cost\": 3.23e-06, \"time\": 0.0355}, \"0eeb372802\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.033600000000000005}, \"0effe9b1dc\": {\"quality\": 0.40216666666666673, \"cost\": 8.4e-07, \"time\": 0.0149}, \"0f7faf684d\": {\"quality\": 0.52195, \"cost\": 3.83e-06, \"time\": 0.0413}, \"0fcec544e3\": {\"quality\": 0.5064500000000001, \"cost\": 1.9799999999999997e-06, \"time\": 0.0286}, \"0ff126ebf8\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"112d9a3421\": {\"quality\": 0.455475, \"cost\": 3.2000000000000003e-06, \"time\": 0.0379}, \"114a097c53\": {\"quality\": 0.43270000000000003, \"cost\": 2.4e-06, \"time\": 0.0224}, \"116334cd72\": {\"quality\": 0.5290250000000001, \"cost\": 4.84e-06, \"time\": 0.045399999999999996}, \"1175ee37e6\": {\"quality\": 0.5367, \"cost\": 1.11e-06, \"time\": 0.0157}, \"11a66478dc\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"11bc996d48\": {\"quality\": 0.5648333333333334, \"cost\": 2.7e-06, \"time\": 0.030799999999999998}, \"11debf9fc0\": {\"quality\": 0.467775, \"cost\": 2.43e-06, \"time\": 0.033299999999999996}, \"123fb650fb\": {\"quality\": 0.429175, \"cost\": 2.2799999999999998e-06, \"time\": 0.0256}, \"1274c21076\": {\"quality\": 0.463975, \"cost\": 3.44e-06, \"time\": 0.0341}, \"127af50739\": {\"quality\": 0.44705, \"cost\": 2.3600000000000003e-06, \"time\": 0.0293}, \"12addbf5e2\": {\"quality\": 0.3989, \"cost\": 1.6799999999999998e-06, \"time\": 0.0202}, \"133ee5023f\": {\"quality\": 0.54595, \"cost\": 2.96e-06, \"time\": 0.025099999999999997}, \"1368e1c78e\": {\"quality\": 0.4021666666666666, \"cost\": 8.4e-07, \"time\": 0.0149}, \"13a009fe0c\": {\"quality\": 0.5399750000000001, \"cost\": 4.24e-06, \"time\": 0.0363}, \"13da306f84\": {\"quality\": 0.4969666666666666, \"cost\": 3.32e-06, \"time\": 0.0277}, \"13f2b9c25b\": {\"quality\": 0.517325, \"cost\": 1.98e-06, \"time\": 0.0319}, \"13f75f9bd0\": {\"quality\": 0.4392333333333333, \"cost\": 1.92e-06, \"time\": 0.023}, \"1404e0aa35\": {\"quality\": 0.513525, \"cost\": 2.99e-06, \"time\": 0.0327}, \"140ededb41\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"142f3a7c70\": {\"quality\": 0.4763, \"cost\": 1.47e-06, \"time\": 0.015}, \"1468dddecc\": {\"quality\": 0.5318, \"cost\": 1.95e-06, \"time\": 0.020999999999999998}, \"14d19a01e2\": {\"quality\": 0.4134, \"cost\": 1.6799999999999998e-06, \"time\": 0.0235}, \"1624bb5302\": {\"quality\": 0.4166666666666667, \"cost\": 8.4e-07, \"time\": 0.0182}, \"1636e0833b\": {\"quality\": 0.37633333333333335, \"cost\": 6e-07, \"time\": 0.0154}, \"1658296f3a\": {\"quality\": 0.41340000000000005, \"cost\": 1.6799999999999998e-06, \"time\": 0.0235}, \"16f351273f\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"171e6ae293\": {\"quality\": 0.5611333333333334, \"cost\": 3.64e-06, \"time\": 0.034199999999999994}, \"176da24f53\": {\"quality\": 0.5291, \"cost\": 2.12e-06, \"time\": 0.0165}, \"179379555f\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"17c928174f\": {\"quality\": 0.5422666666666667, \"cost\": 1.6200000000000002e-06, \"time\": 0.026}, \"181c91d1be\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.030999999999999996}, \"18368684cd\": {\"quality\": 0.4134, \"cost\": 1.6799999999999998e-06, \"time\": 0.0235}, \"183743e76e\": {\"quality\": 0.43596666666666667, \"cost\": 2.76e-06, \"time\": 0.0283}, \"18f55750b0\": {\"quality\": 0.39892500000000003, \"cost\": 2.04e-06, \"time\": 0.0228}, \"19563b057d\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.0229}, \"1957127275\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.0229}, \"197bb53f10\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"199fd1fbf2\": {\"quality\": 0.40375, \"cost\": 4.8e-07, \"time\": 0.0123}, \"19b40e0271\": {\"quality\": 0.513525, \"cost\": 2.99e-06, \"time\": 0.0327}, \"19e3db7fe7\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.0336}, \"1ad856985f\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"1adec2dca2\": {\"quality\": 0.47247500000000003, \"cost\": 3.68e-06, \"time\": 0.0303}, \"1b04a2b184\": {\"quality\": 0.5304500000000001, \"cost\": 4.07e-06, \"time\": 0.037500000000000006}, \"1b28439bd7\": {\"quality\": 0.5724666666666667, \"cost\": 3.88e-06, \"time\": 0.0304}, \"1beb2fac62\": {\"quality\": 0.4969666666666666, \"cost\": 3.32e-06, \"time\": 0.0277}, \"1c347e4d91\": {\"quality\": 0.5053, \"cost\": 1.47e-06, \"time\": 0.021599999999999998}, \"1c3882926e\": {\"quality\": 0.4484, \"cost\": 2.19e-06, \"time\": 0.033800000000000004}, \"1cc6d9efb6\": {\"quality\": 0.5837, \"cost\": 4.7200000000000005e-06, \"time\": 0.03899999999999999}, \"1ce3d77039\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"1ce99cf2c8\": {\"quality\": 0.5050250000000001, \"cost\": 2.7500000000000004e-06, \"time\": 0.0365}, \"1d26090364\": {\"quality\": 0.5318, \"cost\": 1.95e-06, \"time\": 0.020999999999999998}, \"1d87f97e62\": {\"quality\": 0.6403333333333333, \"cost\": 3.26e-06, \"time\": 0.0335}, \"1da2369719\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"1e18e60895\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"1e1bf7e88b\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"1e8b3521f8\": {\"quality\": 0.5680999999999999, \"cost\": 1.86e-06, \"time\": 0.0255}, \"1f5e8c9e9a\": {\"quality\": 0.463975, \"cost\": 3.44e-06, \"time\": 0.034100000000000005}, \"2018bef45f\": {\"quality\": 0.5147333333333334, \"cost\": 2.48e-06, \"time\": 0.025699999999999997}, \"2066966577\": {\"quality\": 0.463975, \"cost\": 3.44e-06, \"time\": 0.0341}, \"2075ff1d04\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"2080b60a57\": {\"quality\": 0.448475, \"cost\": 1.5899999999999998e-06, \"time\": 0.021400000000000002}, \"208a98f514\": {\"quality\": 0.5800000000000001, \"cost\": 3.62e-06, \"time\": 0.03609999999999999}, \"20e10af7d4\": {\"quality\": 0.47872499999999996, \"cost\": 1.8299999999999998e-06, \"time\": 0.0242}, \"20e2c0b057\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"211b89b4cd\": {\"quality\": 0.49876666666666675, \"cost\": 3.15e-06, \"time\": 0.0322}, \"2153174e2d\": {\"quality\": 0.45792499999999997, \"cost\": 2.3600000000000003e-06, \"time\": 0.0326}, \"21b2b8ebd1\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"21b2df8512\": {\"quality\": 0.48889999999999995, \"cost\": 2.24e-06, \"time\": 0.0262}, \"21bed16a7d\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"227246dff8\": {\"quality\": 0.40375, \"cost\": 4.8e-07, \"time\": 0.0123}, \"227c30d349\": {\"quality\": 0.38766666666666666, \"cost\": 8.4e-07, \"time\": 0.011600000000000001}, \"228687831a\": {\"quality\": 0.4101333333333333, \"cost\": 2.52e-06, \"time\": 0.0288}, \"23075b2a6e\": {\"quality\": 0.4649666666666667, \"cost\": 1.23e-06, \"time\": 0.018799999999999997}, \"23566f15ab\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"2370cebb10\": {\"quality\": 0.45935000000000004, \"cost\": 1.59e-06, \"time\": 0.0247}, \"2386b03c4c\": {\"quality\": 0.4484, \"cost\": 2.19e-06, \"time\": 0.033800000000000004}, \"24957f3a43\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.0197}, \"24c122de4e\": {\"quality\": 0.4101333333333334, \"cost\": 2.5199999999999996e-06, \"time\": 0.0288}, \"24f76747b9\": {\"quality\": 0.4649666666666667, \"cost\": 1.23e-06, \"time\": 0.018799999999999997}, \"2609bfd616\": {\"quality\": 0.4858, \"cost\": 2.84e-06, \"time\": 0.0283}, \"260ab3e966\": {\"quality\": 0.420675, \"cost\": 2.04e-06, \"time\": 0.0294}, \"2728c8eb6a\": {\"quality\": 0.42146666666666666, \"cost\": 2.76e-06, \"time\": 0.025}, \"27971eaaf5\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.0229}, \"27ba0964b2\": {\"quality\": 0.40375, \"cost\": 4.8e-07, \"time\": 0.0123}, \"27daa50458\": {\"quality\": 0.5508500000000001, \"cost\": 2.12e-06, \"time\": 0.019799999999999998}, \"28369b2421\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.0263}, \"28421e6d62\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.022399999999999996}, \"2848c42f91\": {\"quality\": 0.4649666666666667, \"cost\": 1.2299999999999999e-06, \"time\": 0.0188}, \"28a638bb6e\": {\"quality\": 0.45935000000000004, \"cost\": 1.59e-06, \"time\": 0.0247}, \"290947fe5a\": {\"quality\": 0.45935, \"cost\": 1.5899999999999998e-06, \"time\": 0.0247}, \"2936c3e43e\": {\"quality\": 0.5304500000000001, \"cost\": 4.07e-06, \"time\": 0.0375}, \"293ec5edca\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"294e541235\": {\"quality\": 0.51495, \"cost\": 1.11e-06, \"time\": 0.0124}, \"295ed5e759\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.0197}, \"2960431101\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"29892d8468\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.0336}, \"299a0aeb65\": {\"quality\": 0.590875, \"cost\": 3.62e-06, \"time\": 0.0394}, \"29bf3c0a3b\": {\"quality\": 0.45690000000000003, \"cost\": 2.43e-06, \"time\": 0.03}, \"29c8c693e2\": {\"quality\": 0.52195, \"cost\": 3.830000000000001e-06, \"time\": 0.041299999999999996}, \"2a7d15f4a7\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"2ae24e0124\": {\"quality\": 0.514875, \"cost\": 2.82e-06, \"time\": 0.0372}, \"2b5679d248\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"2b5ab72a55\": {\"quality\": 0.365, \"cost\": 1.2e-07, \"time\": 0.0064}, \"2b82a67eb1\": {\"quality\": 0.45363333333333333, \"cost\": 9.9e-07, \"time\": 0.0226}, \"2bcbffdf85\": {\"quality\": 0.45935000000000004, \"cost\": 1.59e-06, \"time\": 0.0247}, \"2bd39ee744\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"2bf38d797f\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"2c1640adf7\": {\"quality\": 0.4794666666666667, \"cost\": 1.23e-06, \"time\": 0.0221}, \"2c5cf9eb26\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.023599999999999996}, \"2c87313a93\": {\"quality\": 0.47240000000000004, \"cost\": 4.28e-06, \"time\": 0.0427}, \"2c9a9f94c4\": {\"quality\": 0.426725, \"cost\": 3.1199999999999998e-06, \"time\": 0.030900000000000004}, \"2d3bbc2d23\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"2d7f1dbd4b\": {\"quality\": 0.470225, \"cost\": 1.59e-06, \"time\": 0.027999999999999997}, \"2de113167b\": {\"quality\": 0.494225, \"cost\": 3.68e-06, \"time\": 0.0369}, \"2de3eb2c19\": {\"quality\": 0.6592, \"cost\": 3.52e-06, \"time\": 0.0278}, \"2e02b71061\": {\"quality\": 0.4166666666666667, \"cost\": 8.4e-07, \"time\": 0.0182}, \"2e30394ac6\": {\"quality\": 0.4135, \"cost\": 1.08e-06, \"time\": 0.011099999999999999}, \"2e5d071f21\": {\"quality\": 0.45792499999999997, \"cost\": 2.3600000000000003e-06, \"time\": 0.0326}, \"2e9c5cc9bf\": {\"quality\": 0.5291, \"cost\": 2.12e-06, \"time\": 0.0165}, \"2ec4bec1a3\": {\"quality\": 0.455475, \"cost\": 3.2000000000000003e-06, \"time\": 0.0379}, \"2f1573da80\": {\"quality\": 0.429175, \"cost\": 2.2799999999999998e-06, \"time\": 0.0256}, \"2fc0cb3592\": {\"quality\": 0.43596666666666667, \"cost\": 2.76e-06, \"time\": 0.0283}, \"2fd9cd426a\": {\"quality\": 0.48335, \"cost\": 3.68e-06, \"time\": 0.033600000000000005}, \"3019af79b3\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"302c1d97fc\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027699999999999995}, \"303b467574\": {\"quality\": 0.5244, \"cost\": 2.99e-06, \"time\": 0.036}, \"3058b1f1f8\": {\"quality\": 0.4013, \"cost\": 1.8e-06, \"time\": 0.0299}, \"30ae4cbe91\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"30e3ff1d17\": {\"quality\": 0.47729999999999995, \"cost\": 2.6e-06, \"time\": 0.0321}, \"30f20c8fe6\": {\"quality\": 0.513525, \"cost\": 2.99e-06, \"time\": 0.0327}, \"316759d191\": {\"quality\": 0.463975, \"cost\": 3.44e-06, \"time\": 0.034100000000000005}, \"3177802176\": {\"quality\": 0.365, \"cost\": 1.2e-07, \"time\": 0.0064}, \"3184f977a8\": {\"quality\": 0.39287500000000003, \"cost\": 9.6e-07, \"time\": 0.0213}, \"3194e440cf\": {\"quality\": 0.4762, \"cost\": 2.07e-06, \"time\": 0.027399999999999997}, \"3197ad4faf\": {\"quality\": 0.37633333333333335, \"cost\": 6e-07, \"time\": 0.0154}, \"31a423a3bf\": {\"quality\": 0.382, \"cost\": 4.8e-07, \"time\": 0.009000000000000001}, \"321e17afbd\": {\"quality\": 0.46642500000000003, \"cost\": 2.6e-06, \"time\": 0.0288}, \"32b101d807\": {\"quality\": 0.538875, \"cost\": 4.9100000000000004e-06, \"time\": 0.0461}, \"332a350ea2\": {\"quality\": 0.4134, \"cost\": 1.6799999999999998e-06, \"time\": 0.0235}, \"33459cd29c\": {\"quality\": 0.47492500000000004, \"cost\": 2.84e-06, \"time\": 0.025}, \"33a187e74f\": {\"quality\": 0.525825, \"cost\": 2.22e-06, \"time\": 0.0281}, \"34026bb5cc\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"3511b5e1d0\": {\"quality\": 0.39899999999999997, \"cost\": 1.08e-06, \"time\": 0.0078}, \"3513e54767\": {\"quality\": 0.54595, \"cost\": 2.96e-06, \"time\": 0.025099999999999997}, \"353f0cb1ac\": {\"quality\": 0.5508500000000001, \"cost\": 2.12e-06, \"time\": 0.019799999999999998}, \"3550bf88cb\": {\"quality\": 0.53425, \"cost\": 3.06e-06, \"time\": 0.036699999999999997}, \"35610fb420\": {\"quality\": 0.5536, \"cost\": 1.86e-06, \"time\": 0.022199999999999998}, \"357267e14b\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"36011c7606\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"362d480d6d\": {\"quality\": 0.463975, \"cost\": 3.44e-06, \"time\": 0.034100000000000005}, \"363209b6e7\": {\"quality\": 0.48890000000000006, \"cost\": 2.24e-06, \"time\": 0.026199999999999998}, \"3637084f91\": {\"quality\": 0.5147333333333334, \"cost\": 2.48e-06, \"time\": 0.025699999999999997}, \"36b17c40f3\": {\"quality\": 0.39885000000000004, \"cost\": 1.3199999999999999e-06, \"time\": 0.0176}, \"36c66671ee\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.023599999999999996}, \"372e8b5f4f\": {\"quality\": 0.4134, \"cost\": 1.6799999999999998e-06, \"time\": 0.0235}, \"375ed248fe\": {\"quality\": 0.5147333333333334, \"cost\": 2.48e-06, \"time\": 0.025699999999999997}, \"377b8b0bcc\": {\"quality\": 0.45935, \"cost\": 1.5899999999999998e-06, \"time\": 0.0247}, \"37bd28f2c9\": {\"quality\": 0.5244, \"cost\": 2.99e-06, \"time\": 0.036}, \"38075bb01f\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"38567d6a43\": {\"quality\": 0.4763, \"cost\": 1.47e-06, \"time\": 0.015}, \"389a99ab21\": {\"quality\": 0.46532500000000004, \"cost\": 3.27e-06, \"time\": 0.0386}, \"389c54cbca\": {\"quality\": 0.54595, \"cost\": 2.96e-06, \"time\": 0.025099999999999997}, \"38ec11cf7b\": {\"quality\": 0.45690000000000003, \"cost\": 2.43e-06, \"time\": 0.03}, \"3980f20caa\": {\"quality\": 0.48090000000000005, \"cost\": 4.52e-06, \"time\": 0.038900000000000004}, \"39cd4ca402\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.018299999999999997}, \"3a34b24c41\": {\"quality\": 0.4762, \"cost\": 2.07e-06, \"time\": 0.0274}, \"3b2e8075ea\": {\"quality\": 0.5680999999999999, \"cost\": 1.86e-06, \"time\": 0.0255}, \"3b3676521a\": {\"quality\": 0.5955, \"cost\": 5.47e-06, \"time\": 0.048799999999999996}, \"3b3a6bf087\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"3b4bde0121\": {\"quality\": 0.38766666666666666, \"cost\": 8.4e-07, \"time\": 0.011600000000000001}, \"3b57530a56\": {\"quality\": 0.64505, \"cost\": 2.51e-06, \"time\": 0.0237}, \"3c206c89f3\": {\"quality\": 0.47485, \"cost\": 3.44e-06, \"time\": 0.037399999999999996}, \"3cbab8082e\": {\"quality\": 0.523375, \"cost\": 3.06e-06, \"time\": 0.0334}, \"3d71c4dd2c\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.026899999999999997}, \"3ea15ac20c\": {\"quality\": 0.390425, \"cost\": 1.8e-06, \"time\": 0.026600000000000002}, \"3f1a58aec9\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"3f2321bb08\": {\"quality\": 0.365, \"cost\": 1.2e-07, \"time\": 0.0064}, \"3f3ef494b0\": {\"quality\": 0.43596666666666667, \"cost\": 2.76e-06, \"time\": 0.0283}, \"3f62c3fbfc\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"3f88dd99f7\": {\"quality\": 0.5121, \"cost\": 1.88e-06, \"time\": 0.0203}, \"3fa747af9a\": {\"quality\": 0.3908333333333333, \"cost\": 6e-07, \"time\": 0.0187}, \"40104c813f\": {\"quality\": 0.522025, \"cost\": 3.23e-06, \"time\": 0.028899999999999995}, \"403b05da2d\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027699999999999995}, \"4043815a3e\": {\"quality\": 0.5206, \"cost\": 4.000000000000001e-06, \"time\": 0.0368}, \"409ff67607\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.018299999999999997}, \"412c065b83\": {\"quality\": 0.4102333333333333, \"cost\": 1.92e-06, \"time\": 0.016399999999999998}, \"4171fbac5c\": {\"quality\": 0.47485, \"cost\": 3.44e-06, \"time\": 0.037399999999999996}, \"4191118787\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"41d5b97871\": {\"quality\": 0.43923333333333336, \"cost\": 1.92e-06, \"time\": 0.023}, \"41d8845655\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"41ee202cac\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"42082dcd0d\": {\"quality\": 0.45690000000000003, \"cost\": 2.43e-06, \"time\": 0.030000000000000002}, \"42ddd48341\": {\"quality\": 0.537525, \"cost\": 5.0800000000000005e-06, \"time\": 0.0416}, \"4361bc7ea7\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"43afdad250\": {\"quality\": 0.52195, \"cost\": 3.83e-06, \"time\": 0.0413}, \"43c3cf9cb8\": {\"quality\": 0.6592, \"cost\": 3.52e-06, \"time\": 0.0278}, \"43d24fb32a\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027700000000000002}, \"440bc872de\": {\"quality\": 0.45690000000000003, \"cost\": 2.4299999999999996e-06, \"time\": 0.03}, \"44173a9aef\": {\"quality\": 0.5290250000000001, \"cost\": 4.84e-06, \"time\": 0.045399999999999996}, \"44d6af5523\": {\"quality\": 0.6309, \"cost\": 2.25e-06, \"time\": 0.0294}, \"44fe4e4e3e\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"4587a1500c\": {\"quality\": 0.4630666666666667, \"cost\": 2e-06, \"time\": 0.026699999999999998}, \"45ef93b61e\": {\"quality\": 0.4762, \"cost\": 2.07e-06, \"time\": 0.0274}, \"461846a52d\": {\"quality\": 0.42799999999999994, \"cost\": 1.08e-06, \"time\": 0.0144}, \"462e6ff849\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.0336}, \"4630853d32\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"46475b9e75\": {\"quality\": 0.5536, \"cost\": 1.86e-06, \"time\": 0.022199999999999998}, \"46654a1f32\": {\"quality\": 0.5837, \"cost\": 4.7200000000000005e-06, \"time\": 0.039}, \"466a3036b2\": {\"quality\": 0.47382500000000005, \"cost\": 3.51e-06, \"time\": 0.0348}, \"466d4d16dd\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.022399999999999996}, \"46a35022d8\": {\"quality\": 0.49795, \"cost\": 8.7e-07, \"time\": 0.0162}, \"46ed68152d\": {\"quality\": 0.5002333333333333, \"cost\": 2.48e-06, \"time\": 0.0224}, \"46edc488a4\": {\"quality\": 0.39885000000000004, \"cost\": 1.3199999999999999e-06, \"time\": 0.0176}, \"476a12876c\": {\"quality\": 0.4392333333333333, \"cost\": 1.92e-06, \"time\": 0.023}, \"48043e2304\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"488645cbd9\": {\"quality\": 0.4425, \"cost\": 7.2e-07, \"time\": 0.0118}, \"49009a3b57\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.022899999999999997}, \"4909061216\": {\"quality\": 0.5304500000000001, \"cost\": 4.07e-06, \"time\": 0.037500000000000006}, \"49107972df\": {\"quality\": 0.390425, \"cost\": 1.8e-06, \"time\": 0.026600000000000002}, \"49731b1ccd\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"498f146004\": {\"quality\": 0.5715, \"cost\": 3.3800000000000002e-06, \"time\": 0.0399}, \"49ad844bd2\": {\"quality\": 0.4847, \"cost\": 3.51e-06, \"time\": 0.0381}, \"49ca727e49\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"4a4a960a82\": {\"quality\": 0.48889999999999995, \"cost\": 2.24e-06, \"time\": 0.026199999999999998}, \"4a92372986\": {\"quality\": 0.40216666666666673, \"cost\": 8.4e-07, \"time\": 0.0149}, \"4aa7e8fde6\": {\"quality\": 0.45363333333333333, \"cost\": 9.9e-07, \"time\": 0.022600000000000002}, \"4aafd39d76\": {\"quality\": 0.5536, \"cost\": 1.86e-06, \"time\": 0.022199999999999998}, \"4ace1cfad1\": {\"quality\": 0.463975, \"cost\": 3.44e-06, \"time\": 0.0341}, \"4ad1952206\": {\"quality\": 0.48889999999999995, \"cost\": 2.24e-06, \"time\": 0.0262}, \"4b59f40131\": {\"quality\": 0.4762, \"cost\": 2.07e-06, \"time\": 0.027399999999999997}, \"4b86a1c038\": {\"quality\": 0.46642500000000003, \"cost\": 2.6e-06, \"time\": 0.0288}, \"4b92a26754\": {\"quality\": 0.365, \"cost\": 1.2e-07, \"time\": 0.0064}, \"4bc4528402\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"4c158a1a4a\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.0197}, \"4c954323e3\": {\"quality\": 0.5800000000000001, \"cost\": 3.62e-06, \"time\": 0.03609999999999999}, \"4d8bcf8ae2\": {\"quality\": 0.365, \"cost\": 1.2e-07, \"time\": 0.0064}, \"4d91e8a27b\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"4dd3635bc3\": {\"quality\": 0.6403333333333333, \"cost\": 3.26e-06, \"time\": 0.0335}, \"4dd96bd18f\": {\"quality\": 0.47240000000000004, \"cost\": 4.28e-06, \"time\": 0.0427}, \"4e298ee0d4\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.029500000000000002}, \"4e3443a0f9\": {\"quality\": 0.538875, \"cost\": 4.9100000000000004e-06, \"time\": 0.0461}, \"4e4b9db2b8\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"4e6509f614\": {\"quality\": 0.43270000000000003, \"cost\": 2.4e-06, \"time\": 0.0224}, \"4e6a83e751\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"4e8d8e527a\": {\"quality\": 0.38756666666666667, \"cost\": 1.4399999999999998e-06, \"time\": 0.024}, \"4ef333ab21\": {\"quality\": 0.4166666666666667, \"cost\": 8.4e-07, \"time\": 0.0182}, \"4f16545711\": {\"quality\": 0.39890000000000003, \"cost\": 1.6799999999999998e-06, \"time\": 0.020200000000000003}, \"4f8cca1195\": {\"quality\": 0.59795, \"cost\": 4.63e-06, \"time\": 0.0435}, \"500860eaa2\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"50701b505e\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.026899999999999997}, \"50bc87e9cc\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.0229}, \"50c03be77c\": {\"quality\": 0.3908333333333333, \"cost\": 6e-07, \"time\": 0.0187}, \"510375edad\": {\"quality\": 0.382, \"cost\": 4.8e-07, \"time\": 0.009000000000000001}, \"512fdb607c\": {\"quality\": 0.52195, \"cost\": 3.830000000000001e-06, \"time\": 0.0413}, \"51583a901c\": {\"quality\": 0.59795, \"cost\": 4.63e-06, \"time\": 0.0435}, \"521314dab6\": {\"quality\": 0.54595, \"cost\": 2.96e-06, \"time\": 0.025099999999999997}, \"5241bf401b\": {\"quality\": 0.41340000000000005, \"cost\": 1.6799999999999998e-06, \"time\": 0.0235}, \"526878b5eb\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"52c1cba6ce\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.040799999999999996}, \"52e5d0f4fb\": {\"quality\": 0.3908333333333333, \"cost\": 6e-07, \"time\": 0.0187}, \"52f041a70e\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"5307496302\": {\"quality\": 0.4762, \"cost\": 2.0699999999999997e-06, \"time\": 0.0274}, \"53869388bb\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"53d2932c4f\": {\"quality\": 0.5114666666666666, \"cost\": 3.32e-06, \"time\": 0.031}, \"5474247f91\": {\"quality\": 0.48573333333333335, \"cost\": 2.48e-06, \"time\": 0.0191}, \"557d2cf7ba\": {\"quality\": 0.48573333333333335, \"cost\": 2.48e-06, \"time\": 0.0191}, \"559c7120c5\": {\"quality\": 0.513525, \"cost\": 2.99e-06, \"time\": 0.0327}, \"55c8aa8935\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.0336}, \"56a29a28c5\": {\"quality\": 0.41585, \"cost\": 1.5599999999999999e-06, \"time\": 0.0138}, \"56b39eb1d6\": {\"quality\": 0.5785750000000001, \"cost\": 4.39e-06, \"time\": 0.044}, \"5703697dbd\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"5718f2ed80\": {\"quality\": 0.5680999999999999, \"cost\": 1.86e-06, \"time\": 0.0255}, \"572a02a59a\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"572c2df793\": {\"quality\": 0.439975, \"cost\": 1.3499999999999998e-06, \"time\": 0.0252}, \"5750713a41\": {\"quality\": 0.4392333333333333, \"cost\": 1.92e-06, \"time\": 0.023}, \"57757ef15e\": {\"quality\": 0.5002333333333333, \"cost\": 2.48e-06, \"time\": 0.0224}, \"5793d14bbe\": {\"quality\": 0.5314749999999999, \"cost\": 4.000000000000001e-06, \"time\": 0.0401}, \"579a915ed2\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"579c81bbe0\": {\"quality\": 0.5291, \"cost\": 2.12e-06, \"time\": 0.0165}, \"57bed1722f\": {\"quality\": 0.41585, \"cost\": 1.5599999999999999e-06, \"time\": 0.0138}, \"585ba6d20b\": {\"quality\": 0.5328999999999999, \"cost\": 3.23e-06, \"time\": 0.0322}, \"589a1cea79\": {\"quality\": 0.47492500000000004, \"cost\": 2.84e-06, \"time\": 0.025}, \"59006532b4\": {\"quality\": 0.49876666666666675, \"cost\": 3.15e-06, \"time\": 0.0322}, \"59326c4e00\": {\"quality\": 0.5291, \"cost\": 2.12e-06, \"time\": 0.0165}, \"596f0ed542\": {\"quality\": 0.47485000000000005, \"cost\": 3.44e-06, \"time\": 0.0374}, \"5971ba4e0d\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"59e0117b7d\": {\"quality\": 0.46642500000000003, \"cost\": 2.6e-06, \"time\": 0.0288}, \"59f515d0da\": {\"quality\": 0.48889999999999995, \"cost\": 2.24e-06, \"time\": 0.0262}, \"59f887b67c\": {\"quality\": 0.47492500000000004, \"cost\": 2.84e-06, \"time\": 0.025}, \"5a22920db4\": {\"quality\": 0.5367, \"cost\": 1.11e-06, \"time\": 0.0157}, \"5a35020d45\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"5aa71bb88a\": {\"quality\": 0.40375, \"cost\": 4.8e-07, \"time\": 0.0123}, \"5b10fbdbe1\": {\"quality\": 0.5399750000000001, \"cost\": 4.24e-06, \"time\": 0.0363}, \"5b4ad39a9e\": {\"quality\": 0.48889999999999995, \"cost\": 2.24e-06, \"time\": 0.0262}, \"5bade9eb85\": {\"quality\": 0.523375, \"cost\": 3.06e-06, \"time\": 0.0334}, \"5be16744bf\": {\"quality\": 0.4969666666666666, \"cost\": 3.32e-06, \"time\": 0.0277}, \"5c0db11303\": {\"quality\": 0.37633333333333335, \"cost\": 6e-07, \"time\": 0.0154}, \"5c53feccd9\": {\"quality\": 0.48090000000000005, \"cost\": 4.52e-06, \"time\": 0.038900000000000004}, \"5c77c7c2b2\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"5d072194b8\": {\"quality\": 0.47729999999999995, \"cost\": 2.6e-06, \"time\": 0.0321}, \"5d79b50feb\": {\"quality\": 0.5724666666666667, \"cost\": 3.88e-06, \"time\": 0.030399999999999996}, \"5dc216cd6b\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.030999999999999996}, \"5dd68c1b8f\": {\"quality\": 0.5367, \"cost\": 1.11e-06, \"time\": 0.0157}, \"5de4a882c1\": {\"quality\": 0.6497666666666667, \"cost\": 4.27e-06, \"time\": 0.037599999999999995}, \"5deeeb223f\": {\"quality\": 0.5244, \"cost\": 2.99e-06, \"time\": 0.036}, \"5e2f03b962\": {\"quality\": 0.40375, \"cost\": 4.8e-07, \"time\": 0.0123}, \"5ea2fab380\": {\"quality\": 0.4376, \"cost\": 1.5599999999999999e-06, \"time\": 0.0171}, \"5eb3bb525b\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.022399999999999996}, \"5ec3832817\": {\"quality\": 0.513525, \"cost\": 2.99e-06, \"time\": 0.0327}, \"5f0199e07b\": {\"quality\": 0.382, \"cost\": 4.8e-07, \"time\": 0.009000000000000001}, \"5f37b3902b\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027699999999999995}, \"60cb623c53\": {\"quality\": 0.42074999999999996, \"cost\": 7.2e-07, \"time\": 0.0085}, \"612e546d71\": {\"quality\": 0.40980000000000005, \"cost\": 2.04e-06, \"time\": 0.0261}, \"6160bfb439\": {\"quality\": 0.40735000000000005, \"cost\": 2.88e-06, \"time\": 0.031400000000000004}, \"6178f33808\": {\"quality\": 0.4484, \"cost\": 2.19e-06, \"time\": 0.033800000000000004}, \"619b48dde9\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"6268ac658c\": {\"quality\": 0.5206, \"cost\": 4.000000000000001e-06, \"time\": 0.0368}, \"628f34aace\": {\"quality\": 0.5064500000000001, \"cost\": 1.9799999999999997e-06, \"time\": 0.0286}, \"630d1ecda0\": {\"quality\": 0.40137500000000004, \"cost\": 1.2e-06, \"time\": 0.0175}, \"63a0aaebed\": {\"quality\": 0.51495, \"cost\": 1.11e-06, \"time\": 0.0124}, \"63f392465f\": {\"quality\": 0.45363333333333333, \"cost\": 9.9e-07, \"time\": 0.0226}, \"6527f214c3\": {\"quality\": 0.4021666666666666, \"cost\": 8.4e-07, \"time\": 0.0149}, \"652c0f4bdf\": {\"quality\": 0.47247500000000003, \"cost\": 3.68e-06, \"time\": 0.0303}, \"6533c85913\": {\"quality\": 0.47382500000000005, \"cost\": 3.51e-06, \"time\": 0.0348}, \"65627426e0\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"65801893b4\": {\"quality\": 0.463975, \"cost\": 3.44e-06, \"time\": 0.0341}, \"65b76da9c6\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.022399999999999996}, \"65be1c1306\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"65e0216208\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"65eee615d7\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"66277da52f\": {\"quality\": 0.470225, \"cost\": 1.59e-06, \"time\": 0.027999999999999997}, \"66750c0934\": {\"quality\": 0.6403333333333333, \"cost\": 3.26e-06, \"time\": 0.0335}, \"66776ec181\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"67632141f6\": {\"quality\": 0.41585, \"cost\": 1.5599999999999999e-06, \"time\": 0.0138}, \"677deb302a\": {\"quality\": 0.467775, \"cost\": 2.43e-06, \"time\": 0.0333}, \"67868fcff6\": {\"quality\": 0.5837, \"cost\": 4.7200000000000005e-06, \"time\": 0.03899999999999999}, \"67aad9ea16\": {\"quality\": 0.45690000000000003, \"cost\": 2.4299999999999996e-06, \"time\": 0.03}, \"67bab6732d\": {\"quality\": 0.5082000000000001, \"cost\": 4.16e-06, \"time\": 0.0363}, \"67fe399cf1\": {\"quality\": 0.5147333333333334, \"cost\": 2.48e-06, \"time\": 0.025699999999999997}, \"6846bd8fb3\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"68583552fb\": {\"quality\": 0.39890000000000003, \"cost\": 1.6799999999999998e-06, \"time\": 0.020200000000000003}, \"689e327daf\": {\"quality\": 0.382, \"cost\": 4.8e-07, \"time\": 0.009000000000000001}, \"69b3b67de6\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"69bf3f6ba0\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.0224}, \"69f90e610f\": {\"quality\": 0.52195, \"cost\": 3.830000000000001e-06, \"time\": 0.0413}, \"6a022c3f73\": {\"quality\": 0.42799999999999994, \"cost\": 1.08e-06, \"time\": 0.0144}, \"6a10c53ad8\": {\"quality\": 0.588425, \"cost\": 4.46e-06, \"time\": 0.044700000000000004}, \"6a6348f69d\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"6a8726145c\": {\"quality\": 0.4762, \"cost\": 2.0699999999999997e-06, \"time\": 0.0274}, \"6a8a675442\": {\"quality\": 0.4762, \"cost\": 2.0699999999999997e-06, \"time\": 0.0274}, \"6aac59742a\": {\"quality\": 0.46540000000000004, \"cost\": 2.67e-06, \"time\": 0.0262}, \"6ac193c88f\": {\"quality\": 0.420675, \"cost\": 2.04e-06, \"time\": 0.0294}, \"6ae9e9de0b\": {\"quality\": 0.5244, \"cost\": 2.99e-06, \"time\": 0.036}, \"6b0c585f5c\": {\"quality\": 0.420675, \"cost\": 2.04e-06, \"time\": 0.0294}, \"6b3c16def2\": {\"quality\": 0.5367, \"cost\": 1.11e-06, \"time\": 0.0157}, \"6c05c47050\": {\"quality\": 0.4649666666666667, \"cost\": 1.23e-06, \"time\": 0.0188}, \"6c3667811b\": {\"quality\": 0.41340000000000005, \"cost\": 1.6799999999999998e-06, \"time\": 0.0235}, \"6cc813aa68\": {\"quality\": 0.5291, \"cost\": 2.12e-06, \"time\": 0.0165}, \"6cd78cac7e\": {\"quality\": 0.48890000000000006, \"cost\": 2.24e-06, \"time\": 0.026199999999999998}, \"6d20c6ace0\": {\"quality\": 0.5724666666666667, \"cost\": 3.88e-06, \"time\": 0.0304}, \"6d67c56ba6\": {\"quality\": 0.466425, \"cost\": 2.6e-06, \"time\": 0.0288}, \"6db70dc3b6\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"6e0690f576\": {\"quality\": 0.42074999999999996, \"cost\": 7.2e-07, \"time\": 0.0085}, \"6e3db7ec5e\": {\"quality\": 0.47485, \"cost\": 3.44e-06, \"time\": 0.037399999999999996}, \"6e62bbb47f\": {\"quality\": 0.45085000000000003, \"cost\": 1.35e-06, \"time\": 0.028499999999999998}, \"6e859bfae6\": {\"quality\": 0.439975, \"cost\": 1.3499999999999998e-06, \"time\": 0.0252}, \"6e93514f45\": {\"quality\": 0.4102333333333333, \"cost\": 1.92e-06, \"time\": 0.016399999999999998}, \"6eae47102b\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.0263}, \"6ecf93c479\": {\"quality\": 0.47946666666666665, \"cost\": 1.2299999999999999e-06, \"time\": 0.022099999999999998}, \"6ef3b7127e\": {\"quality\": 0.6592, \"cost\": 3.52e-06, \"time\": 0.0278}, \"6f323f80c7\": {\"quality\": 0.365, \"cost\": 1.2e-07, \"time\": 0.0064}, \"6f60a05c33\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"6fbdd8b57c\": {\"quality\": 0.4021666666666666, \"cost\": 8.4e-07, \"time\": 0.0149}, \"6fd6046c4b\": {\"quality\": 0.45935, \"cost\": 1.5899999999999998e-06, \"time\": 0.0247}, \"6fe0b3f929\": {\"quality\": 0.42146666666666666, \"cost\": 2.76e-06, \"time\": 0.025}, \"6ff4f667f8\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"700474dfbd\": {\"quality\": 0.47946666666666665, \"cost\": 1.2299999999999999e-06, \"time\": 0.022099999999999998}, \"700ab1d309\": {\"quality\": 0.48573333333333335, \"cost\": 2.48e-06, \"time\": 0.0191}, \"7040e83d52\": {\"quality\": 0.6497666666666667, \"cost\": 4.27e-06, \"time\": 0.037599999999999995}, \"704209377f\": {\"quality\": 0.46777500000000005, \"cost\": 2.43e-06, \"time\": 0.033299999999999996}, \"70b666e371\": {\"quality\": 0.39890000000000003, \"cost\": 1.68e-06, \"time\": 0.020200000000000003}, \"70c850e039\": {\"quality\": 0.5318, \"cost\": 1.95e-06, \"time\": 0.020999999999999998}, \"7112a7e64c\": {\"quality\": 0.64505, \"cost\": 2.51e-06, \"time\": 0.0237}, \"7114013f0c\": {\"quality\": 0.45363333333333333, \"cost\": 9.9e-07, \"time\": 0.0226}, \"715070d0ca\": {\"quality\": 0.4098, \"cost\": 2.04e-06, \"time\": 0.026099999999999998}, \"71b615468b\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027700000000000002}, \"71ed893462\": {\"quality\": 0.47485000000000005, \"cost\": 3.44e-06, \"time\": 0.0374}, \"722d41b2f8\": {\"quality\": 0.3989, \"cost\": 1.6799999999999998e-06, \"time\": 0.0202}, \"723fd5589a\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"7250da0f41\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027700000000000002}, \"7260a96349\": {\"quality\": 0.4013, \"cost\": 1.8e-06, \"time\": 0.0299}, \"7347cf0308\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"736e652158\": {\"quality\": 0.46785, \"cost\": 1.8299999999999998e-06, \"time\": 0.020900000000000002}, \"738364d6a2\": {\"quality\": 0.45555, \"cost\": 2.6e-06, \"time\": 0.025500000000000002}, \"739b1f81dc\": {\"quality\": 0.5367, \"cost\": 1.11e-06, \"time\": 0.0157}, \"73c6240c29\": {\"quality\": 0.448475, \"cost\": 1.5899999999999998e-06, \"time\": 0.021400000000000002}, \"73fc2767b9\": {\"quality\": 0.4649666666666667, \"cost\": 1.2299999999999999e-06, \"time\": 0.0188}, \"7445d99939\": {\"quality\": 0.43596666666666667, \"cost\": 2.76e-06, \"time\": 0.0283}, \"7466a5f424\": {\"quality\": 0.537525, \"cost\": 5.0800000000000005e-06, \"time\": 0.0416}, \"74cc4b1bc4\": {\"quality\": 0.4762, \"cost\": 2.07e-06, \"time\": 0.027399999999999997}, \"74d7f64b8c\": {\"quality\": 0.46785, \"cost\": 1.8299999999999998e-06, \"time\": 0.020900000000000002}, \"751869cbec\": {\"quality\": 0.513525, \"cost\": 2.99e-06, \"time\": 0.0327}, \"7524905580\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"752d9649f2\": {\"quality\": 0.5484, \"cost\": 5.0800000000000005e-06, \"time\": 0.044899999999999995}, \"7558c9722d\": {\"quality\": 0.47485, \"cost\": 3.44e-06, \"time\": 0.0374}, \"75ca9cd4f8\": {\"quality\": 0.365, \"cost\": 2.4e-07, \"time\": 0.0128}, \"7604c0aa13\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027699999999999995}, \"765dbc6ad5\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"7707e6e7e3\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"7765576286\": {\"quality\": 0.4847, \"cost\": 3.51e-06, \"time\": 0.0381}, \"77983b6105\": {\"quality\": 0.382, \"cost\": 4.8e-07, \"time\": 0.009000000000000001}, \"77b5740025\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.022899999999999997}, \"77c02b00c1\": {\"quality\": 0.48335, \"cost\": 3.68e-06, \"time\": 0.033600000000000005}, \"77c6a9703a\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"77f293b737\": {\"quality\": 0.5785750000000001, \"cost\": 4.39e-06, \"time\": 0.044}, \"7801da66b9\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"7862ea67cb\": {\"quality\": 0.39885000000000004, \"cost\": 1.3199999999999999e-06, \"time\": 0.0176}, \"786e5d0af5\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"795d119bc7\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"7989343d94\": {\"quality\": 0.4166666666666667, \"cost\": 8.4e-07, \"time\": 0.0182}, \"79fad58f07\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"7a207b42a8\": {\"quality\": 0.4183, \"cost\": 2.2799999999999998e-06, \"time\": 0.0223}, \"7a58d3472b\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"7a7cc658c8\": {\"quality\": 0.52195, \"cost\": 3.830000000000001e-06, \"time\": 0.041299999999999996}, \"7b024a2966\": {\"quality\": 0.513525, \"cost\": 2.99e-06, \"time\": 0.0327}, \"7b6dc3702e\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"7b6f44618e\": {\"quality\": 0.4917750000000001, \"cost\": 4.52e-06, \"time\": 0.0422}, \"7b74b23910\": {\"quality\": 0.5611333333333334, \"cost\": 3.64e-06, \"time\": 0.034199999999999994}, \"7b9cc96081\": {\"quality\": 0.365, \"cost\": 1.2e-07, \"time\": 0.0064}, \"7c45c61d8d\": {\"quality\": 0.467775, \"cost\": 2.43e-06, \"time\": 0.033299999999999996}, \"7c89a2b69e\": {\"quality\": 0.53045, \"cost\": 4.07e-06, \"time\": 0.0375}, \"7d44f0959d\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"7d60c38c5c\": {\"quality\": 0.6309, \"cost\": 1.5e-06, \"time\": 0.0196}, \"7d67e14414\": {\"quality\": 0.517325, \"cost\": 1.98e-06, \"time\": 0.0319}, \"7daf7ff182\": {\"quality\": 0.5082000000000001, \"cost\": 4.16e-06, \"time\": 0.0363}, \"7e0ad1c9c1\": {\"quality\": 0.38756666666666667, \"cost\": 1.4399999999999998e-06, \"time\": 0.024}, \"7ed07ad40a\": {\"quality\": 0.5648333333333334, \"cost\": 2.7e-06, \"time\": 0.030799999999999998}, \"7fa67a7656\": {\"quality\": 0.5114666666666666, \"cost\": 3.32e-06, \"time\": 0.031}, \"806881adcb\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"80a1d9c2f3\": {\"quality\": 0.4021666666666666, \"cost\": 8.4e-07, \"time\": 0.0149}, \"80be7df955\": {\"quality\": 0.42074999999999996, \"cost\": 7.2e-07, \"time\": 0.0085}, \"80bf60c422\": {\"quality\": 0.51495, \"cost\": 1.11e-06, \"time\": 0.0124}, \"81333c7a33\": {\"quality\": 0.6592, \"cost\": 5.28e-06, \"time\": 0.0417}, \"813e75210b\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"815e7116df\": {\"quality\": 0.37633333333333335, \"cost\": 6e-07, \"time\": 0.0154}, \"816068ff07\": {\"quality\": 0.439975, \"cost\": 1.3499999999999998e-06, \"time\": 0.0252}, \"81a4f42fd9\": {\"quality\": 0.5121, \"cost\": 1.88e-06, \"time\": 0.0203}, \"828ccea2d3\": {\"quality\": 0.48889999999999995, \"cost\": 2.24e-06, \"time\": 0.026199999999999998}, \"829df73946\": {\"quality\": 0.64505, \"cost\": 2.51e-06, \"time\": 0.0237}, \"831728b179\": {\"quality\": 0.4101333333333334, \"cost\": 2.5199999999999996e-06, \"time\": 0.0288}, \"831e8b8be5\": {\"quality\": 0.4794666666666667, \"cost\": 1.23e-06, \"time\": 0.022099999999999998}, \"8357183895\": {\"quality\": 0.46540000000000004, \"cost\": 2.67e-06, \"time\": 0.0262}, \"8392a6083a\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.033600000000000005}, \"83b244c163\": {\"quality\": 0.5121, \"cost\": 1.88e-06, \"time\": 0.0203}, \"83c9e66ec6\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"842c0d1062\": {\"quality\": 0.45935000000000004, \"cost\": 1.5899999999999998e-06, \"time\": 0.0247}, \"846bed2aa7\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"847fd49235\": {\"quality\": 0.42799999999999994, \"cost\": 1.08e-06, \"time\": 0.0144}, \"84dc98be95\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"8519bef585\": {\"quality\": 0.42074999999999996, \"cost\": 7.2e-07, \"time\": 0.0085}, \"8572c6af3a\": {\"quality\": 0.5244, \"cost\": 2.99e-06, \"time\": 0.036}, \"85c94a5505\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"85e8eaed6e\": {\"quality\": 0.41225, \"cost\": 1.2e-06, \"time\": 0.0208}, \"862183bfb9\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"8668f65f05\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"870e2f87b4\": {\"quality\": 0.587075, \"cost\": 4.63e-06, \"time\": 0.0402}, \"87c1b31c82\": {\"quality\": 0.463975, \"cost\": 3.44e-06, \"time\": 0.0341}, \"88436e05a9\": {\"quality\": 0.455475, \"cost\": 3.2000000000000003e-06, \"time\": 0.0379}, \"887ad124e1\": {\"quality\": 0.5053, \"cost\": 1.47e-06, \"time\": 0.021599999999999998}, \"8886cb3082\": {\"quality\": 0.5328999999999999, \"cost\": 3.23e-06, \"time\": 0.0322}, \"8940398bf1\": {\"quality\": 0.49795, \"cost\": 8.7e-07, \"time\": 0.0162}, \"8941621423\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"8961e4d901\": {\"quality\": 0.4102333333333333, \"cost\": 1.92e-06, \"time\": 0.016399999999999998}, \"8974aa89a0\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"89a289907e\": {\"quality\": 0.590875, \"cost\": 3.62e-06, \"time\": 0.0394}, \"89a35a09b1\": {\"quality\": 0.48714999999999997, \"cost\": 2.67e-06, \"time\": 0.0328}, \"89bc21961a\": {\"quality\": 0.365, \"cost\": 2.4e-07, \"time\": 0.0128}, \"89fbefd150\": {\"quality\": 0.47485, \"cost\": 3.44e-06, \"time\": 0.0374}, \"8a37c82283\": {\"quality\": 0.5611333333333334, \"cost\": 3.64e-06, \"time\": 0.034199999999999994}, \"8aaadb8649\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"8acd758b7f\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"8b721bbc6f\": {\"quality\": 0.48714999999999997, \"cost\": 2.67e-06, \"time\": 0.0328}, \"8b90f4b639\": {\"quality\": 0.4101333333333334, \"cost\": 2.5199999999999996e-06, \"time\": 0.0288}, \"8bbbe0f52a\": {\"quality\": 0.4392333333333333, \"cost\": 1.92e-06, \"time\": 0.023}, \"8bc184f385\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.026899999999999997}, \"8bf5c3eadc\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"8c195addc7\": {\"quality\": 0.5611333333333334, \"cost\": 3.64e-06, \"time\": 0.034199999999999994}, \"8c9881972c\": {\"quality\": 0.4134, \"cost\": 1.68e-06, \"time\": 0.0235}, \"8cf8b81d84\": {\"quality\": 0.38756666666666667, \"cost\": 1.4399999999999998e-06, \"time\": 0.024}, \"8d79e03266\": {\"quality\": 0.6497666666666667, \"cost\": 4.27e-06, \"time\": 0.037599999999999995}, \"8d90814b94\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"8e33fac90f\": {\"quality\": 0.47485, \"cost\": 3.44e-06, \"time\": 0.0374}, \"8e5842ccbd\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.026899999999999997}, \"8f4caddfe6\": {\"quality\": 0.5869666666666667, \"cost\": 3.88e-06, \"time\": 0.033699999999999994}, \"8f4edde3f0\": {\"quality\": 0.4857333333333333, \"cost\": 2.48e-06, \"time\": 0.0191}, \"900a58f984\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"9025e2480f\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"9028588af4\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"9059fd80ad\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"90d5e40c1b\": {\"quality\": 0.42146666666666666, \"cost\": 2.76e-06, \"time\": 0.025}, \"90d9a86a2a\": {\"quality\": 0.49876666666666675, \"cost\": 3.15e-06, \"time\": 0.0322}, \"90ed9312e1\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.0229}, \"90ff13783c\": {\"quality\": 0.5304500000000001, \"cost\": 4.07e-06, \"time\": 0.0375}, \"90ff8eb055\": {\"quality\": 0.426725, \"cost\": 3.1199999999999998e-06, \"time\": 0.030900000000000004}, \"9104e31369\": {\"quality\": 0.53045, \"cost\": 4.07e-06, \"time\": 0.0375}, \"918983323f\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"91beb0cac1\": {\"quality\": 0.382, \"cost\": 4.8e-07, \"time\": 0.009000000000000001}, \"91c800af6b\": {\"quality\": 0.47382500000000005, \"cost\": 3.51e-06, \"time\": 0.0348}, \"91dd8884db\": {\"quality\": 0.46642500000000003, \"cost\": 2.6e-06, \"time\": 0.0288}, \"924f128b3c\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.0229}, \"9288642e53\": {\"quality\": 0.5291, \"cost\": 2.12e-06, \"time\": 0.0165}, \"92c4137fb1\": {\"quality\": 0.46777500000000005, \"cost\": 2.43e-06, \"time\": 0.0333}, \"92c9dcd43b\": {\"quality\": 0.543775, \"cost\": 3.23e-06, \"time\": 0.0355}, \"92f45e5cc7\": {\"quality\": 0.49795, \"cost\": 8.7e-07, \"time\": 0.0162}, \"93011c0821\": {\"quality\": 0.53045, \"cost\": 4.07e-06, \"time\": 0.0375}, \"9303149ba4\": {\"quality\": 0.5050250000000001, \"cost\": 2.7500000000000004e-06, \"time\": 0.0365}, \"933b4d17dd\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"94010928c6\": {\"quality\": 0.42799999999999994, \"cost\": 1.08e-06, \"time\": 0.0144}, \"9403809e44\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"943baaea0c\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"94569f177a\": {\"quality\": 0.4649666666666667, \"cost\": 1.23e-06, \"time\": 0.018799999999999997}, \"9466542023\": {\"quality\": 0.3876666666666666, \"cost\": 8.4e-07, \"time\": 0.0116}, \"947e28ef2e\": {\"quality\": 0.4166666666666667, \"cost\": 8.4e-07, \"time\": 0.0182}, \"94ac356663\": {\"quality\": 0.5114666666666666, \"cost\": 3.32e-06, \"time\": 0.031}, \"94dff9a424\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"9508356a2e\": {\"quality\": 0.4763, \"cost\": 1.47e-06, \"time\": 0.015}, \"956bdcc254\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.0224}, \"9594b0c783\": {\"quality\": 0.5053, \"cost\": 1.47e-06, \"time\": 0.021599999999999998}, \"964c671f18\": {\"quality\": 0.587075, \"cost\": 4.63e-06, \"time\": 0.0402}, \"9679fe2b69\": {\"quality\": 0.5053, \"cost\": 1.47e-06, \"time\": 0.021599999999999998}, \"968fc95038\": {\"quality\": 0.5053, \"cost\": 1.47e-06, \"time\": 0.0216}, \"96b487c724\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"96e85f9af4\": {\"quality\": 0.38766666666666666, \"cost\": 8.4e-07, \"time\": 0.011600000000000001}, \"96f87d6483\": {\"quality\": 0.53045, \"cost\": 4.07e-06, \"time\": 0.0375}, \"972c83b002\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.023599999999999996}, \"975bc44958\": {\"quality\": 0.4794666666666667, \"cost\": 1.23e-06, \"time\": 0.022099999999999998}, \"977a4d6b6b\": {\"quality\": 0.5484, \"cost\": 5.0800000000000005e-06, \"time\": 0.044899999999999995}, \"97ad4cd41a\": {\"quality\": 0.513525, \"cost\": 2.99e-06, \"time\": 0.0327}, \"97bc30bd83\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"97e1d0db92\": {\"quality\": 0.39890000000000003, \"cost\": 1.68e-06, \"time\": 0.020200000000000003}, \"981da9ba40\": {\"quality\": 0.45690000000000003, \"cost\": 2.43e-06, \"time\": 0.03}, \"98c1ea89f3\": {\"quality\": 0.45935000000000004, \"cost\": 1.5899999999999998e-06, \"time\": 0.0247}, \"98eca2c65c\": {\"quality\": 0.4630666666666667, \"cost\": 2e-06, \"time\": 0.0267}, \"98ecf1a157\": {\"quality\": 0.49795, \"cost\": 8.7e-07, \"time\": 0.0162}, \"9927dc270b\": {\"quality\": 0.5244, \"cost\": 2.99e-06, \"time\": 0.036}, \"99546d91e4\": {\"quality\": 0.5422666666666667, \"cost\": 1.6200000000000002e-06, \"time\": 0.026}, \"99cb0ba736\": {\"quality\": 0.4134, \"cost\": 1.68e-06, \"time\": 0.0235}, \"99e44ab9b2\": {\"quality\": 0.40980000000000005, \"cost\": 2.04e-06, \"time\": 0.026099999999999998}, \"9a0145c9b5\": {\"quality\": 0.4794666666666667, \"cost\": 1.23e-06, \"time\": 0.022099999999999998}, \"9a57ea3f89\": {\"quality\": 0.40980000000000005, \"cost\": 2.04e-06, \"time\": 0.0261}, \"9a8420a0b3\": {\"quality\": 0.4098, \"cost\": 2.04e-06, \"time\": 0.026099999999999998}, \"9aa32e6c96\": {\"quality\": 0.47485, \"cost\": 3.44e-06, \"time\": 0.0374}, \"9aa4abfb50\": {\"quality\": 0.39899999999999997, \"cost\": 7.2e-07, \"time\": 0.0052}, \"9ada932bf5\": {\"quality\": 0.5244, \"cost\": 2.99e-06, \"time\": 0.036}, \"9b6d4915f3\": {\"quality\": 0.43270000000000003, \"cost\": 3.6e-06, \"time\": 0.0336}, \"9bae5bafc1\": {\"quality\": 0.5869666666666667, \"cost\": 3.88e-06, \"time\": 0.033699999999999994}, \"9c549db0a7\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"9c595a2bc9\": {\"quality\": 0.466425, \"cost\": 2.6e-06, \"time\": 0.0288}, \"9c85f8cfcb\": {\"quality\": 0.51495, \"cost\": 1.11e-06, \"time\": 0.0124}, \"9c8cc46e6c\": {\"quality\": 0.51495, \"cost\": 1.11e-06, \"time\": 0.0124}, \"9c97d35a30\": {\"quality\": 0.5680999999999999, \"cost\": 1.86e-06, \"time\": 0.0255}, \"9ca354a53e\": {\"quality\": 0.39890000000000003, \"cost\": 1.68e-06, \"time\": 0.020200000000000003}, \"9ce2c3fd98\": {\"quality\": 0.4135, \"cost\": 1.08e-06, \"time\": 0.0111}, \"9d18cd0737\": {\"quality\": 0.5318, \"cost\": 1.95e-06, \"time\": 0.020999999999999998}, \"9d7142e7b4\": {\"quality\": 0.5244, \"cost\": 2.99e-06, \"time\": 0.036}, \"9d778daa24\": {\"quality\": 0.467775, \"cost\": 2.43e-06, \"time\": 0.0333}, \"9e06360bc9\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"9fb157be35\": {\"quality\": 0.48573333333333335, \"cost\": 2.48e-06, \"time\": 0.0191}, \"9fc44fdeb1\": {\"quality\": 0.45555, \"cost\": 2.6e-06, \"time\": 0.025500000000000002}, \"9ffaa26d5a\": {\"quality\": 0.48890000000000006, \"cost\": 2.24e-06, \"time\": 0.026199999999999998}, \"a041e7777a\": {\"quality\": 0.41340000000000005, \"cost\": 1.6799999999999998e-06, \"time\": 0.0235}, \"a0b81be5b4\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"a0c85d260e\": {\"quality\": 0.5955, \"cost\": 5.47e-06, \"time\": 0.048799999999999996}, \"a0dc9f50ac\": {\"quality\": 0.43596666666666667, \"cost\": 2.76e-06, \"time\": 0.0283}, \"a14c507393\": {\"quality\": 0.4134, \"cost\": 1.68e-06, \"time\": 0.0235}, \"a1881eb481\": {\"quality\": 0.46777500000000005, \"cost\": 2.43e-06, \"time\": 0.0333}, \"a1bb32e6a1\": {\"quality\": 0.4021666666666666, \"cost\": 8.4e-07, \"time\": 0.0149}, \"a2347e8e9e\": {\"quality\": 0.4794666666666667, \"cost\": 1.23e-06, \"time\": 0.022099999999999998}, \"a2aa082d14\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027700000000000002}, \"a2cd339ad9\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.040799999999999996}, \"a2fd03e6a5\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"a31e87d7cb\": {\"quality\": 0.4425, \"cost\": 7.2e-07, \"time\": 0.0118}, \"a344b2d79a\": {\"quality\": 0.52195, \"cost\": 3.830000000000001e-06, \"time\": 0.0413}, \"a3c0ea3342\": {\"quality\": 0.47729999999999995, \"cost\": 2.6e-06, \"time\": 0.0321}, \"a456d75fef\": {\"quality\": 0.38766666666666666, \"cost\": 8.4e-07, \"time\": 0.011600000000000001}, \"a457f6c300\": {\"quality\": 0.4135, \"cost\": 1.08e-06, \"time\": 0.0111}, \"a4767e7679\": {\"quality\": 0.39890000000000003, \"cost\": 1.68e-06, \"time\": 0.020200000000000003}, \"a47de025c8\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"a4ad96343d\": {\"quality\": 0.46642500000000003, \"cost\": 2.6e-06, \"time\": 0.0288}, \"a515a9c8cc\": {\"quality\": 0.4376, \"cost\": 1.5599999999999999e-06, \"time\": 0.0171}, \"a5949b76ec\": {\"quality\": 0.42146666666666666, \"cost\": 2.76e-06, \"time\": 0.025}, \"a5ae4dfe66\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.022899999999999997}, \"a60dd076b8\": {\"quality\": 0.4392333333333333, \"cost\": 1.92e-06, \"time\": 0.023}, \"a6297a6c56\": {\"quality\": 0.4376, \"cost\": 1.5599999999999999e-06, \"time\": 0.0171}, \"a6460dbb7c\": {\"quality\": 0.64505, \"cost\": 2.51e-06, \"time\": 0.0237}, \"a6796ed686\": {\"quality\": 0.45085000000000003, \"cost\": 1.35e-06, \"time\": 0.028499999999999998}, \"a6d2b05ec8\": {\"quality\": 0.5715, \"cost\": 3.3800000000000002e-06, \"time\": 0.0399}, \"a717c4c535\": {\"quality\": 0.4917750000000001, \"cost\": 4.52e-06, \"time\": 0.0422}, \"a721cd9ebf\": {\"quality\": 0.48889999999999995, \"cost\": 2.24e-06, \"time\": 0.026199999999999998}, \"a8090787b1\": {\"quality\": 0.45935000000000004, \"cost\": 1.5899999999999998e-06, \"time\": 0.0247}, \"a854343d46\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.022899999999999997}, \"a86b137d7f\": {\"quality\": 0.5508500000000001, \"cost\": 2.12e-06, \"time\": 0.019799999999999998}, \"a88eb1493c\": {\"quality\": 0.5318, \"cost\": 1.95e-06, \"time\": 0.020999999999999998}, \"a88fb984e3\": {\"quality\": 0.52195, \"cost\": 3.830000000000001e-06, \"time\": 0.0413}, \"a94e2e5f57\": {\"quality\": 0.4630666666666667, \"cost\": 2e-06, \"time\": 0.026699999999999998}, \"a95b4a6dd0\": {\"quality\": 0.42146666666666666, \"cost\": 2.76e-06, \"time\": 0.025}, \"a9621ea4e6\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.0197}, \"a96e22379d\": {\"quality\": 0.46777500000000005, \"cost\": 2.43e-06, \"time\": 0.033299999999999996}, \"a9721a0a50\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.030999999999999996}, \"aa08180e36\": {\"quality\": 0.4858, \"cost\": 2.84e-06, \"time\": 0.0283}, \"aa38702a02\": {\"quality\": 0.42799999999999994, \"cost\": 1.08e-06, \"time\": 0.0144}, \"aa8187c023\": {\"quality\": 0.39890000000000003, \"cost\": 1.6799999999999998e-06, \"time\": 0.020200000000000003}, \"aadbfc418b\": {\"quality\": 0.44250000000000006, \"cost\": 1.08e-06, \"time\": 0.0177}, \"ab1c706436\": {\"quality\": 0.46532500000000004, \"cost\": 3.27e-06, \"time\": 0.0386}, \"ab43b02cb0\": {\"quality\": 0.5082000000000001, \"cost\": 4.16e-06, \"time\": 0.0363}, \"aba21780bc\": {\"quality\": 0.39885000000000004, \"cost\": 1.3199999999999999e-06, \"time\": 0.0176}, \"abbca95f00\": {\"quality\": 0.5121, \"cost\": 1.88e-06, \"time\": 0.0203}, \"ac208e7a1d\": {\"quality\": 0.5724666666666667, \"cost\": 3.88e-06, \"time\": 0.0304}, \"ac7fcf90e2\": {\"quality\": 0.4098, \"cost\": 2.04e-06, \"time\": 0.0261}, \"ac828ffe70\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"ac9fdc1550\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"ad328d5108\": {\"quality\": 0.365, \"cost\": 3.5999999999999994e-07, \"time\": 0.019200000000000002}, \"ad3efe44c3\": {\"quality\": 0.39899999999999997, \"cost\": 7.2e-07, \"time\": 0.0052}, \"ad48432c22\": {\"quality\": 0.46540000000000004, \"cost\": 2.67e-06, \"time\": 0.0262}, \"ad5187a390\": {\"quality\": 0.463975, \"cost\": 3.44e-06, \"time\": 0.034100000000000005}, \"ad6ebbba8d\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"ad90055ef6\": {\"quality\": 0.39899999999999997, \"cost\": 7.2e-07, \"time\": 0.0052}, \"ad97c5cee6\": {\"quality\": 0.3908333333333333, \"cost\": 6e-07, \"time\": 0.0187}, \"adab1e0fb1\": {\"quality\": 0.5508500000000001, \"cost\": 2.12e-06, \"time\": 0.019799999999999998}, \"ae655ec593\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"ae94b172be\": {\"quality\": 0.5002333333333333, \"cost\": 2.48e-06, \"time\": 0.0224}, \"af360c323c\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.018299999999999997}, \"af90567194\": {\"quality\": 0.587075, \"cost\": 4.63e-06, \"time\": 0.0402}, \"b0156bb6d2\": {\"quality\": 0.47485, \"cost\": 3.44e-06, \"time\": 0.0374}, \"b03c31ca45\": {\"quality\": 0.514875, \"cost\": 2.82e-06, \"time\": 0.0372}, \"b0530b98c3\": {\"quality\": 0.46532500000000004, \"cost\": 3.27e-06, \"time\": 0.0386}, \"b0948c05b6\": {\"quality\": 0.5837, \"cost\": 4.7200000000000005e-06, \"time\": 0.03899999999999999}, \"b18168b9c1\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"b1cf8d33e5\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"b1dcd7aa24\": {\"quality\": 0.513525, \"cost\": 2.99e-06, \"time\": 0.0327}, \"b214718d07\": {\"quality\": 0.4649666666666667, \"cost\": 1.23e-06, \"time\": 0.0188}, \"b28925e4b8\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"b2b057ba41\": {\"quality\": 0.5680999999999999, \"cost\": 1.86e-06, \"time\": 0.0255}, \"b2e063499d\": {\"quality\": 0.6497666666666667, \"cost\": 4.27e-06, \"time\": 0.037599999999999995}, \"b3369775dc\": {\"quality\": 0.588425, \"cost\": 4.46e-06, \"time\": 0.044700000000000004}, \"b35bf038c2\": {\"quality\": 0.37633333333333335, \"cost\": 6e-07, \"time\": 0.0154}, \"b363b25367\": {\"quality\": 0.40980000000000005, \"cost\": 2.04e-06, \"time\": 0.0261}, \"b3decd5c2f\": {\"quality\": 0.38756666666666667, \"cost\": 1.4399999999999998e-06, \"time\": 0.024}, \"b3f20b706d\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"b4002173ee\": {\"quality\": 0.43270000000000003, \"cost\": 2.4e-06, \"time\": 0.0224}, \"b45fc30d81\": {\"quality\": 0.45690000000000003, \"cost\": 2.43e-06, \"time\": 0.03}, \"b4a259f6dd\": {\"quality\": 0.44705, \"cost\": 2.3600000000000003e-06, \"time\": 0.0293}, \"b52cdb3c6d\": {\"quality\": 0.4794666666666667, \"cost\": 1.23e-06, \"time\": 0.0221}, \"b56c312eda\": {\"quality\": 0.4847, \"cost\": 3.51e-06, \"time\": 0.0381}, \"b5a02bb8ab\": {\"quality\": 0.40980000000000005, \"cost\": 2.04e-06, \"time\": 0.026099999999999998}, \"b5e2b41c1c\": {\"quality\": 0.5536, \"cost\": 1.86e-06, \"time\": 0.022199999999999998}, \"b64ddb14f9\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"b66118d5f2\": {\"quality\": 0.45690000000000003, \"cost\": 2.43e-06, \"time\": 0.03}, \"b67107a43e\": {\"quality\": 0.46785, \"cost\": 1.8299999999999998e-06, \"time\": 0.020900000000000002}, \"b682a23b89\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"b690a1ddd6\": {\"quality\": 0.40137500000000004, \"cost\": 1.2e-06, \"time\": 0.0175}, \"b796b7ffd3\": {\"quality\": 0.590875, \"cost\": 3.62e-06, \"time\": 0.0394}, \"b7d0e8557f\": {\"quality\": 0.48714999999999997, \"cost\": 2.67e-06, \"time\": 0.0328}, \"b7f203a0bf\": {\"quality\": 0.467775, \"cost\": 2.43e-06, \"time\": 0.0333}, \"b81d5b2bd9\": {\"quality\": 0.5244, \"cost\": 2.99e-06, \"time\": 0.036}, \"b8343f05e1\": {\"quality\": 0.3989, \"cost\": 1.6799999999999998e-06, \"time\": 0.0202}, \"b8ab3d2f25\": {\"quality\": 0.5869666666666667, \"cost\": 3.88e-06, \"time\": 0.033699999999999994}, \"b8b569172f\": {\"quality\": 0.4969666666666666, \"cost\": 3.32e-06, \"time\": 0.0277}, \"b8b91e375d\": {\"quality\": 0.5290250000000001, \"cost\": 4.84e-06, \"time\": 0.045399999999999996}, \"b8c685904d\": {\"quality\": 0.4762, \"cost\": 2.07e-06, \"time\": 0.0274}, \"b8d1903276\": {\"quality\": 0.4649666666666667, \"cost\": 1.23e-06, \"time\": 0.018799999999999997}, \"b91e7fdb29\": {\"quality\": 0.4183, \"cost\": 2.2799999999999998e-06, \"time\": 0.0223}, \"b9770c2261\": {\"quality\": 0.6309, \"cost\": 1.5e-06, \"time\": 0.0196}, \"b9bb1e6f8d\": {\"quality\": 0.588425, \"cost\": 4.46e-06, \"time\": 0.044700000000000004}, \"b9da208432\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"ba3223f6ac\": {\"quality\": 0.428, \"cost\": 1.08e-06, \"time\": 0.0144}, \"bac3d23c31\": {\"quality\": 0.5206, \"cost\": 4.000000000000001e-06, \"time\": 0.0368}, \"bb3ee18de1\": {\"quality\": 0.48890000000000006, \"cost\": 2.24e-06, \"time\": 0.026199999999999998}, \"bb6536b0ab\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.030999999999999996}, \"bb70f60bf1\": {\"quality\": 0.40216666666666673, \"cost\": 8.4e-07, \"time\": 0.0149}, \"bbba9dd6ae\": {\"quality\": 0.64505, \"cost\": 2.51e-06, \"time\": 0.0237}, \"bbfba2f2ee\": {\"quality\": 0.38766666666666666, \"cost\": 8.4e-07, \"time\": 0.011600000000000001}, \"bc3d02f753\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"bc4c1fcc64\": {\"quality\": 0.64505, \"cost\": 2.51e-06, \"time\": 0.0237}, \"bc60556255\": {\"quality\": 0.365, \"cost\": 1.2e-07, \"time\": 0.0064}, \"bcae7c2fc4\": {\"quality\": 0.4762, \"cost\": 2.0699999999999997e-06, \"time\": 0.0274}, \"bcef42e3b0\": {\"quality\": 0.4166666666666667, \"cost\": 8.4e-07, \"time\": 0.0182}, \"bcf4bf7c35\": {\"quality\": 0.40735000000000005, \"cost\": 2.88e-06, \"time\": 0.031400000000000004}, \"bcfb273436\": {\"quality\": 0.5244, \"cost\": 2.99e-06, \"time\": 0.036}, \"bd99b2fb21\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.031}, \"bdf497196b\": {\"quality\": 0.5121, \"cost\": 1.88e-06, \"time\": 0.0203}, \"be2ae88f70\": {\"quality\": 0.4135, \"cost\": 1.08e-06, \"time\": 0.0111}, \"be4740f38f\": {\"quality\": 0.5837, \"cost\": 4.7200000000000005e-06, \"time\": 0.03899999999999999}, \"bed888d4dc\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"bf2a5d2680\": {\"quality\": 0.4134, \"cost\": 1.6799999999999998e-06, \"time\": 0.0235}, \"bf7b0a8dc1\": {\"quality\": 0.45792499999999997, \"cost\": 2.3600000000000003e-06, \"time\": 0.0326}, \"bfed7670ed\": {\"quality\": 0.4969666666666666, \"cost\": 3.32e-06, \"time\": 0.0277}, \"c06b118e65\": {\"quality\": 0.4762, \"cost\": 2.0699999999999997e-06, \"time\": 0.0274}, \"c08a5ad170\": {\"quality\": 0.466425, \"cost\": 2.6e-06, \"time\": 0.0288}, \"c0d53a20de\": {\"quality\": 0.514875, \"cost\": 2.82e-06, \"time\": 0.0372}, \"c10e588987\": {\"quality\": 0.40980000000000005, \"cost\": 2.04e-06, \"time\": 0.026099999999999998}, \"c127509a7a\": {\"quality\": 0.6403333333333333, \"cost\": 3.26e-06, \"time\": 0.0335}, \"c13682c7c7\": {\"quality\": 0.494225, \"cost\": 3.68e-06, \"time\": 0.0369}, \"c13d6e78e9\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"c145482664\": {\"quality\": 0.47485000000000005, \"cost\": 3.44e-06, \"time\": 0.0374}, \"c14ff3144d\": {\"quality\": 0.4425, \"cost\": 7.2e-07, \"time\": 0.0118}, \"c186182658\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"c263c65d0a\": {\"quality\": 0.5064500000000001, \"cost\": 1.9799999999999997e-06, \"time\": 0.0286}, \"c2949aa902\": {\"quality\": 0.5648333333333334, \"cost\": 2.7e-06, \"time\": 0.030799999999999998}, \"c31c9d4d8c\": {\"quality\": 0.4649666666666667, \"cost\": 1.2299999999999999e-06, \"time\": 0.0188}, \"c31e956b35\": {\"quality\": 0.523375, \"cost\": 3.06e-06, \"time\": 0.0334}, \"c339463a25\": {\"quality\": 0.37633333333333335, \"cost\": 6e-07, \"time\": 0.0154}, \"c36b525dde\": {\"quality\": 0.5648333333333334, \"cost\": 2.7e-06, \"time\": 0.030799999999999998}, \"c3d20f33bf\": {\"quality\": 0.5314749999999999, \"cost\": 4.000000000000001e-06, \"time\": 0.0401}, \"c3ec2cec59\": {\"quality\": 0.5724666666666667, \"cost\": 3.88e-06, \"time\": 0.0304}, \"c443a2c1fb\": {\"quality\": 0.4098, \"cost\": 2.04e-06, \"time\": 0.0261}, \"c4a80d19b3\": {\"quality\": 0.5800000000000001, \"cost\": 3.62e-06, \"time\": 0.03609999999999999}, \"c4c2826afd\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"c4c94a5527\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"c4f3e7665d\": {\"quality\": 0.4135, \"cost\": 1.08e-06, \"time\": 0.0111}, \"c58e9652b7\": {\"quality\": 0.39287500000000003, \"cost\": 9.6e-07, \"time\": 0.0213}, \"c5a16b834a\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"c5fbe2076f\": {\"quality\": 0.537525, \"cost\": 5.0800000000000005e-06, \"time\": 0.0416}, \"c6198f364e\": {\"quality\": 0.513525, \"cost\": 2.99e-06, \"time\": 0.0327}, \"c691570715\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.0229}, \"c691a29c42\": {\"quality\": 0.48090000000000005, \"cost\": 4.52e-06, \"time\": 0.038900000000000004}, \"c6a339987c\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"c6a4d256ce\": {\"quality\": 0.52195, \"cost\": 3.83e-06, \"time\": 0.0413}, \"c76222087e\": {\"quality\": 0.4630666666666667, \"cost\": 2e-06, \"time\": 0.026699999999999998}, \"c772ff3704\": {\"quality\": 0.429175, \"cost\": 2.2799999999999998e-06, \"time\": 0.0256}, \"c7d4ff0c05\": {\"quality\": 0.467775, \"cost\": 2.43e-06, \"time\": 0.033299999999999996}, \"c823f7ab29\": {\"quality\": 0.4630666666666667, \"cost\": 2e-06, \"time\": 0.026699999999999998}, \"c82b926689\": {\"quality\": 0.52195, \"cost\": 3.830000000000001e-06, \"time\": 0.0413}, \"c82f834e85\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"c9320068f9\": {\"quality\": 0.40735000000000005, \"cost\": 2.88e-06, \"time\": 0.031400000000000004}, \"c935a33384\": {\"quality\": 0.5484, \"cost\": 5.0800000000000005e-06, \"time\": 0.044899999999999995}, \"c99f3577c7\": {\"quality\": 0.463975, \"cost\": 3.44e-06, \"time\": 0.034100000000000005}, \"c9d32a0a82\": {\"quality\": 0.466425, \"cost\": 2.6e-06, \"time\": 0.0288}, \"ca55c36c3f\": {\"quality\": 0.4794666666666667, \"cost\": 1.23e-06, \"time\": 0.0221}, \"caa7c0bd6b\": {\"quality\": 0.49876666666666675, \"cost\": 3.15e-06, \"time\": 0.0322}, \"cac6b051e9\": {\"quality\": 0.53425, \"cost\": 3.06e-06, \"time\": 0.036699999999999997}, \"cb19d631b2\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"cbb25fb322\": {\"quality\": 0.39287500000000003, \"cost\": 9.6e-07, \"time\": 0.0213}, \"cbb5eb0e74\": {\"quality\": 0.53425, \"cost\": 3.06e-06, \"time\": 0.036699999999999997}, \"cbc32cbeff\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.029500000000000002}, \"cbd4461293\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"cbe2318045\": {\"quality\": 0.5147333333333334, \"cost\": 2.48e-06, \"time\": 0.025699999999999997}, \"cc20ebc768\": {\"quality\": 0.47485, \"cost\": 3.44e-06, \"time\": 0.0374}, \"cc886fe337\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"ccf72745c1\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027700000000000002}, \"cd1d418732\": {\"quality\": 0.5082000000000001, \"cost\": 4.16e-06, \"time\": 0.0363}, \"cd23c79db1\": {\"quality\": 0.48573333333333335, \"cost\": 2.48e-06, \"time\": 0.0191}, \"cd64fbfcd9\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.023599999999999996}, \"cda3e2a4e9\": {\"quality\": 0.4134, \"cost\": 1.68e-06, \"time\": 0.0235}, \"cdc9ce922f\": {\"quality\": 0.4649666666666667, \"cost\": 1.23e-06, \"time\": 0.0188}, \"cdf0df2f51\": {\"quality\": 0.45690000000000003, \"cost\": 2.4299999999999996e-06, \"time\": 0.03}, \"ce281875b4\": {\"quality\": 0.4794666666666667, \"cost\": 1.23e-06, \"time\": 0.022099999999999998}, \"ce4bc5f348\": {\"quality\": 0.42074999999999996, \"cost\": 7.2e-07, \"time\": 0.0085}, \"ce980cf86f\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"cecca90dd2\": {\"quality\": 0.5536, \"cost\": 1.86e-06, \"time\": 0.022199999999999998}, \"cece83de2d\": {\"quality\": 0.4101333333333334, \"cost\": 2.5199999999999996e-06, \"time\": 0.0288}, \"cf51e0a888\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.0229}, \"cf9538faf0\": {\"quality\": 0.5147333333333334, \"cost\": 2.48e-06, \"time\": 0.0257}, \"cf9d2e224c\": {\"quality\": 0.5508500000000001, \"cost\": 2.12e-06, \"time\": 0.019799999999999998}, \"cfd36f3a8c\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.0263}, \"cffa29a6ef\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"d03596c3de\": {\"quality\": 0.5328999999999999, \"cost\": 3.23e-06, \"time\": 0.0322}, \"d0f9633442\": {\"quality\": 0.4102333333333333, \"cost\": 1.92e-06, \"time\": 0.016399999999999998}, \"d192872a51\": {\"quality\": 0.47240000000000004, \"cost\": 4.28e-06, \"time\": 0.0427}, \"d1d953cac7\": {\"quality\": 0.3989, \"cost\": 1.6799999999999998e-06, \"time\": 0.0202}, \"d2164c8c4c\": {\"quality\": 0.5611333333333334, \"cost\": 3.64e-06, \"time\": 0.034199999999999994}, \"d216eab7d8\": {\"quality\": 0.5869666666666667, \"cost\": 3.88e-06, \"time\": 0.033699999999999994}, \"d266c19ac8\": {\"quality\": 0.525825, \"cost\": 2.22e-06, \"time\": 0.0281}, \"d2af24b59e\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.0224}, \"d3a2d50bd7\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"d3db4cf84d\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"d40174fb0b\": {\"quality\": 0.39885000000000004, \"cost\": 1.3199999999999999e-06, \"time\": 0.0176}, \"d43fafa19e\": {\"quality\": 0.41585, \"cost\": 1.5599999999999999e-06, \"time\": 0.0138}, \"d48ead13da\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.022399999999999996}, \"d5016f4538\": {\"quality\": 0.41585, \"cost\": 1.5599999999999999e-06, \"time\": 0.0138}, \"d55e983189\": {\"quality\": 0.5422666666666667, \"cost\": 1.6200000000000002e-06, \"time\": 0.026}, \"d573c2a414\": {\"quality\": 0.45935000000000004, \"cost\": 1.59e-06, \"time\": 0.0247}, \"d58036ba66\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.0224}, \"d59bacbfe0\": {\"quality\": 0.52195, \"cost\": 3.830000000000001e-06, \"time\": 0.041299999999999996}, \"d6040140b9\": {\"quality\": 0.5955, \"cost\": 5.47e-06, \"time\": 0.048799999999999996}, \"d640edd7a7\": {\"quality\": 0.4794666666666667, \"cost\": 1.23e-06, \"time\": 0.0221}, \"d65185c1a4\": {\"quality\": 0.5680999999999999, \"cost\": 1.86e-06, \"time\": 0.0255}, \"d690b6d739\": {\"quality\": 0.4649666666666667, \"cost\": 1.23e-06, \"time\": 0.018799999999999997}, \"d6bd3b66ba\": {\"quality\": 0.4135, \"cost\": 1.08e-06, \"time\": 0.0111}, \"d6c4e48eeb\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.0263}, \"d6c60a5214\": {\"quality\": 0.40375, \"cost\": 4.8e-07, \"time\": 0.0123}, \"d6cbf265ee\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"d73a9aab4e\": {\"quality\": 0.41585, \"cost\": 1.5599999999999999e-06, \"time\": 0.0138}, \"d752c30d07\": {\"quality\": 0.5002333333333333, \"cost\": 2.48e-06, \"time\": 0.0224}, \"d7c0972014\": {\"quality\": 0.5082, \"cost\": 4.16e-06, \"time\": 0.0363}, \"d7f6c0c9d4\": {\"quality\": 0.4762, \"cost\": 2.07e-06, \"time\": 0.0274}, \"d813410e44\": {\"quality\": 0.47946666666666665, \"cost\": 1.2299999999999999e-06, \"time\": 0.022099999999999998}, \"d87eb775da\": {\"quality\": 0.6403333333333333, \"cost\": 3.26e-06, \"time\": 0.0335}, \"d8bab6c09b\": {\"quality\": 0.53045, \"cost\": 4.07e-06, \"time\": 0.0375}, \"d8bcac36e8\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027699999999999995}, \"d96677d8d4\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.026899999999999997}, \"d9e2bb21a3\": {\"quality\": 0.5053, \"cost\": 1.47e-06, \"time\": 0.021599999999999998}, \"daaadadcc9\": {\"quality\": 0.4858, \"cost\": 2.84e-06, \"time\": 0.0283}, \"daf855e065\": {\"quality\": 0.525825, \"cost\": 2.22e-06, \"time\": 0.0281}, \"db6f7259cd\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"db9060cd27\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"dbce95a072\": {\"quality\": 0.5114666666666666, \"cost\": 3.32e-06, \"time\": 0.031}, \"dbe7d818fa\": {\"quality\": 0.3908333333333333, \"cost\": 6e-07, \"time\": 0.0187}, \"dc66bccb1c\": {\"quality\": 0.5002333333333333, \"cost\": 2.48e-06, \"time\": 0.0224}, \"dc90065dea\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"dc9a912501\": {\"quality\": 0.4649666666666667, \"cost\": 1.2299999999999999e-06, \"time\": 0.0188}, \"dd0d70fedd\": {\"quality\": 0.6309, \"cost\": 1.5e-06, \"time\": 0.0196}, \"dd76899626\": {\"quality\": 0.463975, \"cost\": 3.44e-06, \"time\": 0.0341}, \"dd9f5d1ba9\": {\"quality\": 0.45085000000000003, \"cost\": 1.35e-06, \"time\": 0.028499999999999998}, \"de1645053b\": {\"quality\": 0.5314749999999999, \"cost\": 4.000000000000001e-06, \"time\": 0.0401}, \"de18bf45e1\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"de1e56370f\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"deb84ddd06\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"df2bb408cf\": {\"quality\": 0.40216666666666673, \"cost\": 8.4e-07, \"time\": 0.0149}, \"df2ebe2c01\": {\"quality\": 0.45935000000000004, \"cost\": 1.59e-06, \"time\": 0.0247}, \"dfb8aebe38\": {\"quality\": 0.5244, \"cost\": 2.99e-06, \"time\": 0.036}, \"dfce6153aa\": {\"quality\": 0.418225, \"cost\": 2.88e-06, \"time\": 0.0347}, \"dfda94bd2a\": {\"quality\": 0.42074999999999996, \"cost\": 7.2e-07, \"time\": 0.0085}, \"dff452a9ca\": {\"quality\": 0.4102333333333333, \"cost\": 1.92e-06, \"time\": 0.016399999999999998}, \"e02f982a26\": {\"quality\": 0.48889999999999995, \"cost\": 2.24e-06, \"time\": 0.0262}, \"e06701b665\": {\"quality\": 0.4649666666666667, \"cost\": 1.2299999999999999e-06, \"time\": 0.0188}, \"e18abd2ab0\": {\"quality\": 0.4744, \"cost\": 2.24e-06, \"time\": 0.022899999999999997}, \"e1e596ee1b\": {\"quality\": 0.46642500000000003, \"cost\": 2.6e-06, \"time\": 0.0288}, \"e20ba014a1\": {\"quality\": 0.59795, \"cost\": 4.63e-06, \"time\": 0.0435}, \"e223700849\": {\"quality\": 0.4134, \"cost\": 1.68e-06, \"time\": 0.0235}, \"e26c7bfbdb\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.0197}, \"e35e5f81a7\": {\"quality\": 0.4763, \"cost\": 1.47e-06, \"time\": 0.015}, \"e3cdc0d870\": {\"quality\": 0.4021666666666666, \"cost\": 8.4e-07, \"time\": 0.0149}, \"e3df4cf041\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"e3eca9854c\": {\"quality\": 0.41225, \"cost\": 1.2e-06, \"time\": 0.0208}, \"e47dc3abca\": {\"quality\": 0.5082000000000001, \"cost\": 4.16e-06, \"time\": 0.0363}, \"e495ff601f\": {\"quality\": 0.466425, \"cost\": 2.6e-06, \"time\": 0.0288}, \"e4b9d4fb41\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"e510bda989\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"e515bc1935\": {\"quality\": 0.48889999999999995, \"cost\": 2.24e-06, \"time\": 0.026199999999999998}, \"e517cd2222\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"e51b01f418\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"e520dfae5b\": {\"quality\": 0.47872499999999996, \"cost\": 1.8299999999999998e-06, \"time\": 0.0242}, \"e53906f84b\": {\"quality\": 0.4649666666666667, \"cost\": 1.23e-06, \"time\": 0.0188}, \"e53b349cce\": {\"quality\": 0.3989, \"cost\": 1.6799999999999998e-06, \"time\": 0.0202}, \"e5401ed278\": {\"quality\": 0.463975, \"cost\": 3.44e-06, \"time\": 0.034100000000000005}, \"e56a16ca66\": {\"quality\": 0.5367, \"cost\": 1.11e-06, \"time\": 0.0157}, \"e5a2f72b30\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"e5a70a13ac\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"e5fdeb4de9\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"e609601eee\": {\"quality\": 0.39890000000000003, \"cost\": 1.6799999999999998e-06, \"time\": 0.020200000000000003}, \"e6f141cc8f\": {\"quality\": 0.45690000000000003, \"cost\": 2.43e-06, \"time\": 0.030000000000000002}, \"e7525c117a\": {\"quality\": 0.4021666666666666, \"cost\": 8.4e-07, \"time\": 0.0149}, \"e7e94ab7a5\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"e86bd256d6\": {\"quality\": 0.4021666666666666, \"cost\": 8.4e-07, \"time\": 0.0149}, \"e89283a4d9\": {\"quality\": 0.3908333333333333, \"cost\": 6e-07, \"time\": 0.0187}, \"e9befb80e0\": {\"quality\": 0.4101333333333334, \"cost\": 2.5199999999999996e-06, \"time\": 0.0288}, \"ea38031fc1\": {\"quality\": 0.418225, \"cost\": 2.88e-06, \"time\": 0.0347}, \"ea6ecc5653\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"ea8bcb3ae2\": {\"quality\": 0.5837, \"cost\": 4.7200000000000005e-06, \"time\": 0.03899999999999999}, \"ea91a2e78b\": {\"quality\": 0.39890000000000003, \"cost\": 1.68e-06, \"time\": 0.020200000000000003}, \"eac300f0d1\": {\"quality\": 0.45690000000000003, \"cost\": 2.43e-06, \"time\": 0.03}, \"ebaa9b1297\": {\"quality\": 0.49795, \"cost\": 8.7e-07, \"time\": 0.0162}, \"ebbe8b6c4f\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.031}, \"ebdf3abff2\": {\"quality\": 0.6497666666666667, \"cost\": 4.27e-06, \"time\": 0.037599999999999995}, \"ebee1f2761\": {\"quality\": 0.46777500000000005, \"cost\": 2.43e-06, \"time\": 0.033299999999999996}, \"ec8844a5ae\": {\"quality\": 0.365, \"cost\": 1.2e-07, \"time\": 0.0064}, \"ecb5f78f37\": {\"quality\": 0.522025, \"cost\": 3.23e-06, \"time\": 0.028899999999999995}, \"ece7ff5129\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"ed60b8cac5\": {\"quality\": 0.40137500000000004, \"cost\": 1.2e-06, \"time\": 0.0175}, \"ed6b5480a5\": {\"quality\": 0.4376, \"cost\": 1.5599999999999999e-06, \"time\": 0.0171}, \"eda630dc85\": {\"quality\": 0.6403333333333333, \"cost\": 3.26e-06, \"time\": 0.0335}, \"edaaee5ed4\": {\"quality\": 0.54595, \"cost\": 2.96e-06, \"time\": 0.025099999999999997}, \"edb2b764aa\": {\"quality\": 0.5869666666666666, \"cost\": 3.88e-06, \"time\": 0.0337}, \"edc52339db\": {\"quality\": 0.47247500000000003, \"cost\": 3.68e-06, \"time\": 0.0303}, \"ee46042c5d\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.0336}, \"ee855899d8\": {\"quality\": 0.45363333333333333, \"cost\": 9.9e-07, \"time\": 0.0226}, \"eef12d478b\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"ef37b3e0be\": {\"quality\": 0.4183, \"cost\": 2.2799999999999998e-06, \"time\": 0.0223}, \"ef43d497f1\": {\"quality\": 0.48335, \"cost\": 3.68e-06, \"time\": 0.033600000000000005}, \"ef4d4c4a62\": {\"quality\": 0.5869666666666667, \"cost\": 3.88e-06, \"time\": 0.033699999999999994}, \"f0655621af\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"f076b4c9ae\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"f07881a734\": {\"quality\": 0.38756666666666667, \"cost\": 1.44e-06, \"time\": 0.024}, \"f0829510fc\": {\"quality\": 0.418225, \"cost\": 2.88e-06, \"time\": 0.0347}, \"f11eddb4ed\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"f12622d3d7\": {\"quality\": 0.39892500000000003, \"cost\": 2.04e-06, \"time\": 0.0228}, \"f1408da253\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"f1770e7d28\": {\"quality\": 0.517325, \"cost\": 1.98e-06, \"time\": 0.0319}, \"f18cf41929\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"f1bda127f6\": {\"quality\": 0.5114666666666666, \"cost\": 3.32e-06, \"time\": 0.031}, \"f2a2e91541\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.029500000000000002}, \"f2c04ed1c8\": {\"quality\": 0.4376, \"cost\": 1.5599999999999999e-06, \"time\": 0.0171}, \"f2cf5db12d\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.031}, \"f3c7a062f7\": {\"quality\": 0.5422666666666667, \"cost\": 1.6200000000000002e-06, \"time\": 0.026}, \"f42277df7f\": {\"quality\": 0.49795, \"cost\": 8.7e-07, \"time\": 0.0162}, \"f437481e3b\": {\"quality\": 0.5399750000000001, \"cost\": 4.24e-06, \"time\": 0.0363}, \"f487340019\": {\"quality\": 0.4762, \"cost\": 2.07e-06, \"time\": 0.0274}, \"f497b83523\": {\"quality\": 0.5517, \"cost\": 2.6300000000000002e-06, \"time\": 0.0301}, \"f4aa8ffdf5\": {\"quality\": 0.40216666666666673, \"cost\": 8.4e-07, \"time\": 0.0149}, \"f4ab4a73b6\": {\"quality\": 0.48563333333333336, \"cost\": 3.08e-06, \"time\": 0.0315}, \"f4beb148d0\": {\"quality\": 0.5422666666666667, \"cost\": 1.6200000000000002e-06, \"time\": 0.026}, \"f4deb72db6\": {\"quality\": 0.4102333333333333, \"cost\": 1.92e-06, \"time\": 0.016399999999999998}, \"f50a47a0aa\": {\"quality\": 0.47946666666666665, \"cost\": 1.2299999999999999e-06, \"time\": 0.022099999999999998}, \"f5b9a94dcc\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"f5c27e7172\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.018299999999999997}, \"f5e53d963b\": {\"quality\": 0.4376, \"cost\": 1.5599999999999999e-06, \"time\": 0.0171}, \"f5ecac6743\": {\"quality\": 0.466425, \"cost\": 2.6e-06, \"time\": 0.0288}, \"f614235c15\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"f627bff3a1\": {\"quality\": 0.47946666666666665, \"cost\": 1.2299999999999999e-06, \"time\": 0.022099999999999998}, \"f70e54a9a0\": {\"quality\": 0.513525, \"cost\": 2.99e-06, \"time\": 0.0327}, \"f74c5a862e\": {\"quality\": 0.4098, \"cost\": 2.04e-06, \"time\": 0.026099999999999998}, \"f74ec023e4\": {\"quality\": 0.494225, \"cost\": 3.68e-06, \"time\": 0.0369}, \"f783b2b34a\": {\"quality\": 0.4649666666666667, \"cost\": 1.23e-06, \"time\": 0.0188}, \"f7b048bd54\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"f7c4df993e\": {\"quality\": 0.4763, \"cost\": 1.47e-06, \"time\": 0.015}, \"f848813dca\": {\"quality\": 0.448475, \"cost\": 1.5899999999999998e-06, \"time\": 0.021400000000000002}, \"f854533145\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"f8f946b5fb\": {\"quality\": 0.390425, \"cost\": 1.8e-06, \"time\": 0.026600000000000002}, \"f912592c8d\": {\"quality\": 0.5785750000000001, \"cost\": 4.39e-06, \"time\": 0.044}, \"f93d9a2693\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"f99096d89c\": {\"quality\": 0.543775, \"cost\": 3.23e-06, \"time\": 0.0355}, \"f9e8e221f3\": {\"quality\": 0.43596666666666667, \"cost\": 2.76e-06, \"time\": 0.0283}, \"fa38879eab\": {\"quality\": 0.538875, \"cost\": 4.9100000000000004e-06, \"time\": 0.0461}, \"fa5b473f15\": {\"quality\": 0.39892500000000003, \"cost\": 2.04e-06, \"time\": 0.0228}, \"fa7882d46b\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"fa906520d1\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"faabebaa30\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"fb0339a7d0\": {\"quality\": 0.5304500000000001, \"cost\": 4.07e-06, \"time\": 0.0375}, \"fb0e796cd3\": {\"quality\": 0.38756666666666667, \"cost\": 1.4399999999999998e-06, \"time\": 0.024}, \"fb216ad6b3\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"fb29372712\": {\"quality\": 0.5715, \"cost\": 3.3800000000000002e-06, \"time\": 0.0399}, \"fb6216880a\": {\"quality\": 0.522025, \"cost\": 3.23e-06, \"time\": 0.028899999999999995}, \"fbd6c45271\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"fc1fd5bf54\": {\"quality\": 0.47872499999999996, \"cost\": 1.8299999999999998e-06, \"time\": 0.0242}, \"fc4832696b\": {\"quality\": 0.41340000000000005, \"cost\": 1.6799999999999998e-06, \"time\": 0.0235}, \"fccaadfcdb\": {\"quality\": 0.513525, \"cost\": 2.99e-06, \"time\": 0.0327}, \"fcd3d2b250\": {\"quality\": 0.45690000000000003, \"cost\": 2.43e-06, \"time\": 0.030000000000000002}, \"fce38334b2\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.033600000000000005}, \"fd0709359e\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"fddccfbf94\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.030999999999999996}, \"fe2b4d4d8b\": {\"quality\": 0.41225, \"cost\": 1.2e-06, \"time\": 0.0208}, \"fea4734c09\": {\"quality\": 0.54595, \"cost\": 2.96e-06, \"time\": 0.025099999999999997}, \"fef1ca27fa\": {\"quality\": 0.4917750000000001, \"cost\": 4.52e-06, \"time\": 0.0422}, \"ff11cb6a7a\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.0336}, \"ff171e34e2\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.023599999999999996}, \"ff1c958e21\": {\"quality\": 0.51495, \"cost\": 1.11e-06, \"time\": 0.0124}, \"ff8e68049a\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}}" }, { "path": "abacus-research/cheap-priors.json", "content": "{\"00c93aec22\": {\"quality\": 0.426725, \"cost\": 3.1199999999999998e-06, \"time\": 0.030900000000000004}, \"00f4acd0d3\": {\"quality\": 0.49876666666666675, \"cost\": 3.15e-06, \"time\": 0.0322}, \"0121878170\": {\"quality\": 0.630875, \"cost\": 2.611e-05, \"time\": 0.0282}, \"01c2f973ad\": {\"quality\": 0.5318, \"cost\": 1.95e-06, \"time\": 0.020999999999999998}, \"01fca3c717\": {\"quality\": 0.5406666666666666, \"cost\": 1.4060000000000001e-05, \"time\": 0.024999999999999998}, \"02078988c1\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.031}, \"021604dec1\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"0262668df7\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"0267c97b70\": {\"quality\": 0.5648333333333334, \"cost\": 2.7e-06, \"time\": 0.0308}, \"02d6cdecdc\": {\"quality\": 0.5703, \"cost\": 1.582e-05, \"time\": 0.038900000000000004}, \"033ca325e6\": {\"quality\": 0.5508500000000001, \"cost\": 2.12e-06, \"time\": 0.019799999999999998}, \"0364b5e990\": {\"quality\": 0.5594250000000001, \"cost\": 1.582e-05, \"time\": 0.0356}, \"0375ea52c9\": {\"quality\": 0.7468, \"cost\": 1.25e-05, \"time\": 0.0079}, \"038a5f0a62\": {\"quality\": 0.5648333333333334, \"cost\": 2.7e-06, \"time\": 0.030799999999999998}, \"039803b3b1\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.0289}, \"03b972cb56\": {\"quality\": 0.5594250000000001, \"cost\": 1.582e-05, \"time\": 0.03560000000000001}, \"042d933706\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"050b21ce37\": {\"quality\": 0.5161250000000001, \"cost\": 1.4420000000000001e-05, \"time\": 0.0309}, \"0524f42520\": {\"quality\": 0.63795, \"cost\": 2.712e-05, \"time\": 0.032299999999999995}, \"05420351e5\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"057e332ab1\": {\"quality\": 0.5594250000000001, \"cost\": 1.582e-05, \"time\": 0.0356}, \"0646f3f0fb\": {\"quality\": 0.608975, \"cost\": 1.537e-05, \"time\": 0.034199999999999994}, \"06493715cc\": {\"quality\": 0.4763, \"cost\": 1.47e-06, \"time\": 0.015}, \"0659531b94\": {\"quality\": 0.6695333333333333, \"cost\": 1.4000000000000001e-05, \"time\": 0.0275}, \"067ee6e91b\": {\"quality\": 0.49876666666666675, \"cost\": 3.15e-06, \"time\": 0.0322}, \"0695f9b5fc\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"06e94a0f2e\": {\"quality\": 0.7468, \"cost\": 1.25e-05, \"time\": 0.0079}, \"073ef31d23\": {\"quality\": 0.5548, \"cost\": 1.397e-05, \"time\": 0.026199999999999998}, \"078a7e545e\": {\"quality\": 0.6019, \"cost\": 1.436e-05, \"time\": 0.0301}, \"079feb14a8\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.0263}, \"07a3a7daf7\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"08127cd6dd\": {\"quality\": 0.6497666666666667, \"cost\": 4.27e-06, \"time\": 0.037599999999999995}, \"0833133620\": {\"quality\": 0.5536, \"cost\": 1.86e-06, \"time\": 0.022199999999999998}, \"087a2cabc4\": {\"quality\": 0.6174000000000001, \"cost\": 1.6210000000000002e-05, \"time\": 0.042800000000000005}, \"08bf8cc191\": {\"quality\": 0.53045, \"cost\": 4.07e-06, \"time\": 0.0375}, \"08e1802287\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"08f7f63b30\": {\"quality\": 0.563225, \"cost\": 1.481e-05, \"time\": 0.0348}, \"090cd3ef31\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"0947216ece\": {\"quality\": 0.5724666666666667, \"cost\": 3.88e-06, \"time\": 0.0304}, \"096d51f670\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"09791c731b\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"0990c0d4f8\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"0a4c1bbb4a\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.018299999999999997}, \"0ac969dde3\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.040799999999999996}, \"0b4ab72197\": {\"quality\": 0.42146666666666666, \"cost\": 2.76e-06, \"time\": 0.025}, \"0bf9d31691\": {\"quality\": 0.5149333333333334, \"cost\": 1.322e-05, \"time\": 0.0131}, \"0c020b86a3\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.0224}, \"0c6c7fe96a\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"0c81c8996a\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"0cd25da9fe\": {\"quality\": 0.5594250000000001, \"cost\": 1.582e-05, \"time\": 0.0356}, \"0cd78f33d8\": {\"quality\": 0.7081666666666666, \"cost\": 2.5750000000000002e-05, \"time\": 0.0256}, \"0cdc5954dd\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.031}, \"0d53dd53c1\": {\"quality\": 0.5922000000000001, \"cost\": 2.6560000000000003e-05, \"time\": 0.0329}, \"0d8436af32\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.0289}, \"0e38896654\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.0289}, \"0ec672e7c8\": {\"quality\": 0.5304500000000001, \"cost\": 4.07e-06, \"time\": 0.037500000000000006}, \"0ed243f788\": {\"quality\": 0.543775, \"cost\": 3.23e-06, \"time\": 0.0355}, \"0eeb372802\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.033600000000000005}, \"0ef0becc1b\": {\"quality\": 0.6067333333333333, \"cost\": 1.361e-05, \"time\": 0.0236}, \"0fefead197\": {\"quality\": 0.6417499999999999, \"cost\": 2.611e-05, \"time\": 0.0315}, \"0ff126ebf8\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"10d1d4bdeb\": {\"quality\": 0.561875, \"cost\": 1.498e-05, \"time\": 0.0303}, \"114a097c53\": {\"quality\": 0.43270000000000003, \"cost\": 2.4e-06, \"time\": 0.0224}, \"1175ee37e6\": {\"quality\": 0.5367, \"cost\": 1.11e-06, \"time\": 0.0157}, \"11bc996d48\": {\"quality\": 0.5648333333333334, \"cost\": 2.7e-06, \"time\": 0.030799999999999998}, \"11ded03305\": {\"quality\": 0.6067333333333332, \"cost\": 1.361e-05, \"time\": 0.0236}, \"123fb650fb\": {\"quality\": 0.429175, \"cost\": 2.2799999999999998e-06, \"time\": 0.0256}, \"132f6f3946\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"133ee5023f\": {\"quality\": 0.54595, \"cost\": 2.96e-06, \"time\": 0.025099999999999997}, \"13a009fe0c\": {\"quality\": 0.5399750000000001, \"cost\": 4.24e-06, \"time\": 0.0363}, \"13da306f84\": {\"quality\": 0.4969666666666666, \"cost\": 3.32e-06, \"time\": 0.0277}, \"13e717e41e\": {\"quality\": 0.59465, \"cost\": 1.286e-05, \"time\": 0.0138}, \"13f75f9bd0\": {\"quality\": 0.4392333333333333, \"cost\": 1.92e-06, \"time\": 0.023}, \"140ededb41\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"142e59c03f\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.024399999999999998}, \"142f3a7c70\": {\"quality\": 0.4763, \"cost\": 1.47e-06, \"time\": 0.015}, \"1468dddecc\": {\"quality\": 0.5318, \"cost\": 1.95e-06, \"time\": 0.020999999999999998}, \"15af009a01\": {\"quality\": 0.5703, \"cost\": 1.582e-05, \"time\": 0.038900000000000004}, \"15b80a55d3\": {\"quality\": 0.561875, \"cost\": 1.498e-05, \"time\": 0.030299999999999997}, \"1625e624c5\": {\"quality\": 0.58975, \"cost\": 1.3700000000000001e-05, \"time\": 0.0191}, \"16cff1c1e9\": {\"quality\": 0.49682499999999996, \"cost\": 1.358e-05, \"time\": 0.019000000000000003}, \"17407df027\": {\"quality\": 0.5028, \"cost\": 1.526e-05, \"time\": 0.0329}, \"176da24f53\": {\"quality\": 0.5291, \"cost\": 2.12e-06, \"time\": 0.0165}, \"179379555f\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"181c91d1be\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.030999999999999996}, \"183743e76e\": {\"quality\": 0.43596666666666667, \"cost\": 2.76e-06, \"time\": 0.0283}, \"186b58c209\": {\"quality\": 0.5161250000000001, \"cost\": 1.4420000000000001e-05, \"time\": 0.0309}, \"187eace9fe\": {\"quality\": 0.5149333333333334, \"cost\": 1.322e-05, \"time\": 0.0131}, \"190ed2e1b6\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.0289}, \"191aafe1a6\": {\"quality\": 0.6067333333333333, \"cost\": 1.361e-05, \"time\": 0.0236}, \"194919ad28\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.021699999999999997}, \"197bb53f10\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"19ba7d0617\": {\"quality\": 0.7468, \"cost\": 3.7500000000000003e-05, \"time\": 0.023700000000000002}, \"19e3db7fe7\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.0336}, \"1a08cb3f50\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.028900000000000002}, \"1a169179f6\": {\"quality\": 0.646375, \"cost\": 2.7960000000000003e-05, \"time\": 0.040900000000000006}, \"1a71d61ac4\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.0217}, \"1ad856985f\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"1adec2dca2\": {\"quality\": 0.47247500000000003, \"cost\": 3.68e-06, \"time\": 0.0303}, \"1b04a2b184\": {\"quality\": 0.5304500000000001, \"cost\": 4.07e-06, \"time\": 0.037500000000000006}, \"1b28439bd7\": {\"quality\": 0.5724666666666667, \"cost\": 3.88e-06, \"time\": 0.0304}, \"1b2c667b15\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"1b4511eada\": {\"quality\": 0.612775, \"cost\": 1.436e-05, \"time\": 0.0334}, \"1b7e6cad66\": {\"quality\": 0.7030000000000001, \"cost\": 1.426e-05, \"time\": 0.0218}, \"1beb2fac62\": {\"quality\": 0.4969666666666666, \"cost\": 3.32e-06, \"time\": 0.0277}, \"1c347e4d91\": {\"quality\": 0.5053, \"cost\": 1.47e-06, \"time\": 0.021599999999999998}, \"1c35bf4be6\": {\"quality\": 0.610325, \"cost\": 1.52e-05, \"time\": 0.0387}, \"1c4bbf8f7e\": {\"quality\": 0.630875, \"cost\": 2.611e-05, \"time\": 0.0282}, \"1c5f1341f6\": {\"quality\": 0.6067333333333333, \"cost\": 1.361e-05, \"time\": 0.0236}, \"1c71804bec\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.0244}, \"1cc6d9efb6\": {\"quality\": 0.5837, \"cost\": 4.7200000000000005e-06, \"time\": 0.03899999999999999}, \"1ce3d77039\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"1d26090364\": {\"quality\": 0.5318, \"cost\": 1.95e-06, \"time\": 0.020999999999999998}, \"1d87f97e62\": {\"quality\": 0.6403333333333333, \"cost\": 3.26e-06, \"time\": 0.0335}, \"1d90fb8ca6\": {\"quality\": 0.7081666666666666, \"cost\": 2.5750000000000002e-05, \"time\": 0.0256}, \"1da2369719\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"1e18e60895\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"1e1bf7e88b\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"1e8b3521f8\": {\"quality\": 0.5680999999999999, \"cost\": 1.86e-06, \"time\": 0.0255}, \"1f412964ff\": {\"quality\": 0.6067333333333332, \"cost\": 1.361e-05, \"time\": 0.0236}, \"1f72cfb78a\": {\"quality\": 0.5703, \"cost\": 1.582e-05, \"time\": 0.038900000000000004}, \"1fb5d170ad\": {\"quality\": 0.5548, \"cost\": 1.397e-05, \"time\": 0.0262}, \"20180dd292\": {\"quality\": 0.608975, \"cost\": 1.537e-05, \"time\": 0.0342}, \"2018bef45f\": {\"quality\": 0.5147333333333334, \"cost\": 2.48e-06, \"time\": 0.025699999999999997}, \"2075ff1d04\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"208a98f514\": {\"quality\": 0.5800000000000001, \"cost\": 3.62e-06, \"time\": 0.03609999999999999}, \"20904e5c14\": {\"quality\": 0.5439333333333334, \"cost\": 1.322e-05, \"time\": 0.0197}, \"20afc3d539\": {\"quality\": 0.7176, \"cost\": 2.676e-05, \"time\": 0.0297}, \"20e10af7d4\": {\"quality\": 0.47872499999999996, \"cost\": 1.8299999999999998e-06, \"time\": 0.0242}, \"20e2c0b057\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"211b89b4cd\": {\"quality\": 0.49876666666666675, \"cost\": 3.15e-06, \"time\": 0.0322}, \"21386082aa\": {\"quality\": 0.561875, \"cost\": 1.498e-05, \"time\": 0.0303}, \"21b2b8ebd1\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"21b78249a7\": {\"quality\": 0.6308666666666666, \"cost\": 2.536e-05, \"time\": 0.0184}, \"21bed16a7d\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"2200d969d0\": {\"quality\": 0.695925, \"cost\": 2.751e-05, \"time\": 0.03950000000000001}, \"220d008704\": {\"quality\": 0.5548, \"cost\": 1.397e-05, \"time\": 0.0262}, \"2251d21392\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"23566f15ab\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"23a9506d36\": {\"quality\": 0.7468, \"cost\": 1.25e-05, \"time\": 0.0079}, \"24957f3a43\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.0197}, \"2529e2f8b0\": {\"quality\": 0.7468, \"cost\": 1.25e-05, \"time\": 0.0079}, \"252f01ac5b\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.024399999999999998}, \"25fadf0883\": {\"quality\": 0.5703, \"cost\": 1.582e-05, \"time\": 0.038900000000000004}, \"2609bfd616\": {\"quality\": 0.4858, \"cost\": 2.84e-06, \"time\": 0.0283}, \"2629f3e324\": {\"quality\": 0.5632250000000001, \"cost\": 1.481e-05, \"time\": 0.0348}, \"262e4298f9\": {\"quality\": 0.50525, \"cost\": 1.4420000000000001e-05, \"time\": 0.0276}, \"26cc40d3bb\": {\"quality\": 0.5509999999999999, \"cost\": 1.498e-05, \"time\": 0.026999999999999996}, \"2728c8eb6a\": {\"quality\": 0.42146666666666666, \"cost\": 2.76e-06, \"time\": 0.025}, \"27bc52befa\": {\"quality\": 0.5374, \"cost\": 1.49e-05, \"time\": 0.0303}, \"27daa50458\": {\"quality\": 0.5508500000000001, \"cost\": 2.12e-06, \"time\": 0.019799999999999998}, \"2821795e69\": {\"quality\": 0.608975, \"cost\": 1.537e-05, \"time\": 0.034199999999999994}, \"28369b2421\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.0263}, \"28421e6d62\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.022399999999999996}, \"2936c3e43e\": {\"quality\": 0.5304500000000001, \"cost\": 4.07e-06, \"time\": 0.0375}, \"293ec5edca\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"29409d0894\": {\"quality\": 0.59465, \"cost\": 1.286e-05, \"time\": 0.0138}, \"294258298a\": {\"quality\": 0.6067333333333332, \"cost\": 1.361e-05, \"time\": 0.0236}, \"294e541235\": {\"quality\": 0.51495, \"cost\": 1.11e-06, \"time\": 0.0124}, \"295ed5e759\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.0197}, \"2960431101\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"29892d8468\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.0336}, \"299a0aeb65\": {\"quality\": 0.590875, \"cost\": 3.62e-06, \"time\": 0.0394}, \"29ad99e3ed\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.016399999999999998}, \"2a5edac2de\": {\"quality\": 0.5922000000000001, \"cost\": 2.6560000000000003e-05, \"time\": 0.0329}, \"2a7d15f4a7\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"2aa996de6a\": {\"quality\": 0.5837749999999999, \"cost\": 2.572e-05, \"time\": 0.024300000000000002}, \"2ac4fb293f\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"2afeff0083\": {\"quality\": 0.563225, \"cost\": 1.481e-05, \"time\": 0.034800000000000005}, \"2b2bc9568b\": {\"quality\": 0.55235, \"cost\": 1.481e-05, \"time\": 0.0315}, \"2b5679d248\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"2bcf54cda1\": {\"quality\": 0.50525, \"cost\": 1.4420000000000001e-05, \"time\": 0.0276}, \"2bd39ee744\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"2bf38d797f\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"2c4f4f304e\": {\"quality\": 0.68885, \"cost\": 1.325e-05, \"time\": 0.0177}, \"2c5cf9eb26\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.023599999999999996}, \"2c9a9f94c4\": {\"quality\": 0.426725, \"cost\": 3.1199999999999998e-06, \"time\": 0.030900000000000004}, \"2d3bbc2d23\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"2de113167b\": {\"quality\": 0.494225, \"cost\": 3.68e-06, \"time\": 0.0369}, \"2de3eb2c19\": {\"quality\": 0.6592, \"cost\": 3.52e-06, \"time\": 0.0278}, \"2e30394ac6\": {\"quality\": 0.4135, \"cost\": 1.08e-06, \"time\": 0.011099999999999999}, \"2e9c5cc9bf\": {\"quality\": 0.5291, \"cost\": 2.12e-06, \"time\": 0.0165}, \"2f1573da80\": {\"quality\": 0.429175, \"cost\": 2.2799999999999998e-06, \"time\": 0.0256}, \"2f39d78f34\": {\"quality\": 0.6453666666666668, \"cost\": 2.536e-05, \"time\": 0.0217}, \"2fc0cb3592\": {\"quality\": 0.43596666666666667, \"cost\": 2.76e-06, \"time\": 0.0283}, \"2fd9cd426a\": {\"quality\": 0.48335, \"cost\": 3.68e-06, \"time\": 0.033600000000000005}, \"300924ebae\": {\"quality\": 0.6308666666666666, \"cost\": 2.536e-05, \"time\": 0.0184}, \"3019af79b3\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"302c1d97fc\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027699999999999995}, \"30ae4cbe91\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"30c1f9ddf1\": {\"quality\": 0.6067333333333333, \"cost\": 1.361e-05, \"time\": 0.0236}, \"30cd375570\": {\"quality\": 0.6129000000000001, \"cost\": 1.546e-05, \"time\": 0.033}, \"3169782cbb\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.0164}, \"3172fc459a\": {\"quality\": 0.6244750000000001, \"cost\": 1.722e-05, \"time\": 0.0469}, \"318499c14b\": {\"quality\": 0.610325, \"cost\": 1.52e-05, \"time\": 0.0387}, \"31a32be94d\": {\"quality\": 0.6393, \"cost\": 2.695e-05, \"time\": 0.0368}, \"32b101d807\": {\"quality\": 0.538875, \"cost\": 4.9100000000000004e-06, \"time\": 0.0461}, \"32e2c7ad7f\": {\"quality\": 0.61985, \"cost\": 1.537e-05, \"time\": 0.0375}, \"33459cd29c\": {\"quality\": 0.47492500000000004, \"cost\": 2.84e-06, \"time\": 0.025}, \"33a187e74f\": {\"quality\": 0.525825, \"cost\": 2.22e-06, \"time\": 0.0281}, \"33bab4f766\": {\"quality\": 0.612775, \"cost\": 1.436e-05, \"time\": 0.0334}, \"34922140da\": {\"quality\": 0.5703, \"cost\": 1.582e-05, \"time\": 0.038900000000000004}, \"3511b5e1d0\": {\"quality\": 0.39899999999999997, \"cost\": 1.08e-06, \"time\": 0.0078}, \"3513311c2d\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"3513e54767\": {\"quality\": 0.54595, \"cost\": 2.96e-06, \"time\": 0.025099999999999997}, \"353f0cb1ac\": {\"quality\": 0.5508500000000001, \"cost\": 2.12e-06, \"time\": 0.019799999999999998}, \"3550bf88cb\": {\"quality\": 0.53425, \"cost\": 3.06e-06, \"time\": 0.036699999999999997}, \"35610fb420\": {\"quality\": 0.5536, \"cost\": 1.86e-06, \"time\": 0.022199999999999998}, \"357267e14b\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"35baa5c3cc\": {\"quality\": 0.61985, \"cost\": 1.537e-05, \"time\": 0.0375}, \"3637084f91\": {\"quality\": 0.5147333333333334, \"cost\": 2.48e-06, \"time\": 0.025699999999999997}, \"368a497102\": {\"quality\": 0.5406666666666666, \"cost\": 1.4060000000000001e-05, \"time\": 0.025}, \"36c66671ee\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.023599999999999996}, \"37456cb002\": {\"quality\": 0.55235, \"cost\": 1.481e-05, \"time\": 0.0315}, \"3746ea5c03\": {\"quality\": 0.5406666666666666, \"cost\": 1.4060000000000001e-05, \"time\": 0.024999999999999998}, \"375ed248fe\": {\"quality\": 0.5147333333333334, \"cost\": 2.48e-06, \"time\": 0.025699999999999997}, \"377cdf9209\": {\"quality\": 0.563225, \"cost\": 1.481e-05, \"time\": 0.0348}, \"37d4d0f214\": {\"quality\": 0.5703, \"cost\": 1.582e-05, \"time\": 0.0389}, \"37ece7217f\": {\"quality\": 0.59465, \"cost\": 1.286e-05, \"time\": 0.0138}, \"38075bb01f\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"3831d758b1\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.021699999999999997}, \"38567d6a43\": {\"quality\": 0.4763, \"cost\": 1.47e-06, \"time\": 0.015}, \"3875787727\": {\"quality\": 0.55235, \"cost\": 1.481e-05, \"time\": 0.0315}, \"389c54cbca\": {\"quality\": 0.54595, \"cost\": 2.96e-06, \"time\": 0.025099999999999997}, \"3980f20caa\": {\"quality\": 0.48090000000000005, \"cost\": 4.52e-06, \"time\": 0.038900000000000004}, \"3997a836bd\": {\"quality\": 0.608975, \"cost\": 1.537e-05, \"time\": 0.034199999999999994}, \"39ad76f8ce\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.024399999999999998}, \"39c0b7c171\": {\"quality\": 0.6174000000000001, \"cost\": 1.621e-05, \"time\": 0.042800000000000005}, \"39cd4ca402\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.018299999999999997}, \"3a32c98a53\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"3ac7fa4e46\": {\"quality\": 0.646375, \"cost\": 2.7960000000000003e-05, \"time\": 0.040900000000000006}, \"3ad6dcf559\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"3ae0de8663\": {\"quality\": 0.5594250000000001, \"cost\": 1.582e-05, \"time\": 0.03560000000000001}, \"3b2e8075ea\": {\"quality\": 0.5680999999999999, \"cost\": 1.86e-06, \"time\": 0.0255}, \"3b3676521a\": {\"quality\": 0.5955, \"cost\": 5.47e-06, \"time\": 0.048799999999999996}, \"3b57530a56\": {\"quality\": 0.64505, \"cost\": 2.51e-06, \"time\": 0.0237}, \"3b6fbfa11d\": {\"quality\": 0.6129000000000001, \"cost\": 1.546e-05, \"time\": 0.033}, \"3b81215e7a\": {\"quality\": 0.61985, \"cost\": 1.537e-05, \"time\": 0.037500000000000006}, \"3b9f8045d7\": {\"quality\": 0.646375, \"cost\": 2.7960000000000003e-05, \"time\": 0.040900000000000006}, \"3c5857683c\": {\"quality\": 0.5678500000000001, \"cost\": 1.666e-05, \"time\": 0.0442}, \"3cbab8082e\": {\"quality\": 0.523375, \"cost\": 3.06e-06, \"time\": 0.0334}, \"3d21104666\": {\"quality\": 0.6421, \"cost\": 2.62e-05, \"time\": 0.027}, \"3d71c4dd2c\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.026899999999999997}, \"3d9e24215e\": {\"quality\": 0.7030000000000001, \"cost\": 1.426e-05, \"time\": 0.0218}, \"3e7efee65a\": {\"quality\": 0.61985, \"cost\": 1.537e-05, \"time\": 0.0375}, \"3ed0ad20ed\": {\"quality\": 0.5594250000000001, \"cost\": 1.582e-05, \"time\": 0.0356}, \"3f1a58aec9\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"3f2b07cb78\": {\"quality\": 0.6129000000000001, \"cost\": 1.546e-05, \"time\": 0.033}, \"3f3ef494b0\": {\"quality\": 0.43596666666666667, \"cost\": 2.76e-06, \"time\": 0.0283}, \"3f62c3fbfc\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"3f730d8bfe\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.0244}, \"3f8d2ee81f\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"40104c813f\": {\"quality\": 0.522025, \"cost\": 3.23e-06, \"time\": 0.028899999999999995}, \"403b05da2d\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027699999999999995}, \"403f0726fa\": {\"quality\": 0.7468, \"cost\": 2.5e-05, \"time\": 0.0158}, \"4098178354\": {\"quality\": 0.563225, \"cost\": 1.481e-05, \"time\": 0.034800000000000005}, \"409ff67607\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.018299999999999997}, \"40b3b6642c\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"412c065b83\": {\"quality\": 0.4102333333333333, \"cost\": 1.92e-06, \"time\": 0.016399999999999998}, \"4191118787\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"41d5b97871\": {\"quality\": 0.43923333333333336, \"cost\": 1.92e-06, \"time\": 0.023}, \"41d8845655\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"41ee202cac\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"41fe4aee55\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.024399999999999998}, \"42430ea391\": {\"quality\": 0.559425, \"cost\": 1.582e-05, \"time\": 0.0356}, \"42ddd48341\": {\"quality\": 0.537525, \"cost\": 5.0800000000000005e-06, \"time\": 0.0416}, \"42f1e19aa7\": {\"quality\": 0.63795, \"cost\": 2.712e-05, \"time\": 0.032299999999999995}, \"430a2ab32f\": {\"quality\": 0.7176, \"cost\": 2.676e-05, \"time\": 0.0297}, \"4339427ad8\": {\"quality\": 0.513675, \"cost\": 1.526e-05, \"time\": 0.0362}, \"4361bc7ea7\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"43c3cf9cb8\": {\"quality\": 0.6592, \"cost\": 3.52e-06, \"time\": 0.0278}, \"43d24fb32a\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027700000000000002}, \"43e9b39e5c\": {\"quality\": 0.6740250000000001, \"cost\": 1.677e-05, \"time\": 0.0455}, \"44d6af5523\": {\"quality\": 0.6309, \"cost\": 2.25e-06, \"time\": 0.0294}, \"44f189d813\": {\"quality\": 0.55235, \"cost\": 1.481e-05, \"time\": 0.0315}, \"450f45a187\": {\"quality\": 0.5729, \"cost\": 1.286e-05, \"time\": 0.0105}, \"453d0a5097\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.0217}, \"4547ef4c8e\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"461846a52d\": {\"quality\": 0.42799999999999994, \"cost\": 1.08e-06, \"time\": 0.0144}, \"462e6ff849\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.0336}, \"4630853d32\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"46475b9e75\": {\"quality\": 0.5536, \"cost\": 1.86e-06, \"time\": 0.022199999999999998}, \"46654a1f32\": {\"quality\": 0.5837, \"cost\": 4.7200000000000005e-06, \"time\": 0.039}, \"466a3036b2\": {\"quality\": 0.47382500000000005, \"cost\": 3.51e-06, \"time\": 0.0348}, \"466d4d16dd\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.022399999999999996}, \"46ed68152d\": {\"quality\": 0.5002333333333333, \"cost\": 2.48e-06, \"time\": 0.0224}, \"476a12876c\": {\"quality\": 0.4392333333333333, \"cost\": 1.92e-06, \"time\": 0.023}, \"4778401a7a\": {\"quality\": 0.50525, \"cost\": 1.4420000000000001e-05, \"time\": 0.027600000000000003}, \"47f9115b26\": {\"quality\": 0.561875, \"cost\": 1.498e-05, \"time\": 0.0303}, \"48043e2304\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"487f30e740\": {\"quality\": 0.6129, \"cost\": 1.546e-05, \"time\": 0.033}, \"488645cbd9\": {\"quality\": 0.4425, \"cost\": 7.2e-07, \"time\": 0.0118}, \"48bf87f7fe\": {\"quality\": 0.648825, \"cost\": 2.712e-05, \"time\": 0.0356}, \"4909061216\": {\"quality\": 0.5304500000000001, \"cost\": 4.07e-06, \"time\": 0.037500000000000006}, \"49731b1ccd\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"49ad844bd2\": {\"quality\": 0.4847, \"cost\": 3.51e-06, \"time\": 0.0381}, \"49ca727e49\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"4a23d8eff7\": {\"quality\": 0.50525, \"cost\": 1.4420000000000001e-05, \"time\": 0.0276}, \"4a555da784\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"4a5cea8b85\": {\"quality\": 0.6421, \"cost\": 2.62e-05, \"time\": 0.027}, \"4a767339bd\": {\"quality\": 0.57275, \"cost\": 1.498e-05, \"time\": 0.0336}, \"4aafd39d76\": {\"quality\": 0.5536, \"cost\": 1.86e-06, \"time\": 0.022199999999999998}, \"4aca6e5216\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.0164}, \"4b18a647d6\": {\"quality\": 0.630875, \"cost\": 2.611e-05, \"time\": 0.0282}, \"4bc4528402\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"4c158a1a4a\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.0197}, \"4c954323e3\": {\"quality\": 0.5800000000000001, \"cost\": 3.62e-06, \"time\": 0.03609999999999999}, \"4d91e8a27b\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"4dc185389a\": {\"quality\": 0.5656749999999999, \"cost\": 1.397e-05, \"time\": 0.0295}, \"4dd3635bc3\": {\"quality\": 0.6403333333333333, \"cost\": 3.26e-06, \"time\": 0.0335}, \"4dd98ef398\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.021699999999999997}, \"4dfacd0007\": {\"quality\": 0.563225, \"cost\": 1.481e-05, \"time\": 0.0348}, \"4e298ee0d4\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.029500000000000002}, \"4e3443a0f9\": {\"quality\": 0.538875, \"cost\": 4.9100000000000004e-06, \"time\": 0.0461}, \"4e4b9db2b8\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"4e6509f614\": {\"quality\": 0.43270000000000003, \"cost\": 2.4e-06, \"time\": 0.0224}, \"4e6a83e751\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"4e79c8947f\": {\"quality\": 0.58975, \"cost\": 1.3700000000000001e-05, \"time\": 0.0191}, \"4e9504432b\": {\"quality\": 0.560775, \"cost\": 1.565e-05, \"time\": 0.040100000000000004}, \"4e962170dc\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.024399999999999998}, \"4eb0826f21\": {\"quality\": 0.5729, \"cost\": 1.286e-05, \"time\": 0.0105}, \"4ed41bf2e4\": {\"quality\": 0.5837749999999999, \"cost\": 2.572e-05, \"time\": 0.024300000000000002}, \"4f78672528\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.0164}, \"4f8cca1195\": {\"quality\": 0.59795, \"cost\": 4.63e-06, \"time\": 0.0435}, \"500860eaa2\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"50701b505e\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.026899999999999997}, \"51583a901c\": {\"quality\": 0.59795, \"cost\": 4.63e-06, \"time\": 0.0435}, \"51aeaf9f3e\": {\"quality\": 0.7468, \"cost\": 1.25e-05, \"time\": 0.0079}, \"520b52b64c\": {\"quality\": 0.7468, \"cost\": 2.5e-05, \"time\": 0.0158}, \"521314dab6\": {\"quality\": 0.54595, \"cost\": 2.96e-06, \"time\": 0.025099999999999997}, \"5226eb7ff6\": {\"quality\": 0.6067333333333332, \"cost\": 1.361e-05, \"time\": 0.0236}, \"526878b5eb\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"52c1cba6ce\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.040799999999999996}, \"52f041a70e\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"533867574b\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.0217}, \"53869388bb\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"53aefd41e4\": {\"quality\": 0.5439333333333334, \"cost\": 1.322e-05, \"time\": 0.019700000000000002}, \"53d2932c4f\": {\"quality\": 0.5114666666666666, \"cost\": 3.32e-06, \"time\": 0.031}, \"54375d3eba\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.0217}, \"5474247f91\": {\"quality\": 0.48573333333333335, \"cost\": 2.48e-06, \"time\": 0.0191}, \"54993bc472\": {\"quality\": 0.59465, \"cost\": 1.286e-05, \"time\": 0.0138}, \"55358f2285\": {\"quality\": 0.6884, \"cost\": 1.602e-05, \"time\": 0.035699999999999996}, \"5569b4f878\": {\"quality\": 0.68885, \"cost\": 1.325e-05, \"time\": 0.0177}, \"557d2cf7ba\": {\"quality\": 0.48573333333333335, \"cost\": 2.48e-06, \"time\": 0.0191}, \"55c8aa8935\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.0336}, \"55e6bf8f14\": {\"quality\": 0.5406666666666666, \"cost\": 1.4060000000000001e-05, \"time\": 0.024999999999999998}, \"56a0660622\": {\"quality\": 0.58975, \"cost\": 1.3700000000000001e-05, \"time\": 0.0191}, \"56a29a28c5\": {\"quality\": 0.41585, \"cost\": 1.5599999999999999e-06, \"time\": 0.0138}, \"56c4fd5056\": {\"quality\": 0.6308666666666667, \"cost\": 2.536e-05, \"time\": 0.0184}, \"5703697dbd\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"5718f2ed80\": {\"quality\": 0.5680999999999999, \"cost\": 1.86e-06, \"time\": 0.0255}, \"572a02a59a\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"5750713a41\": {\"quality\": 0.4392333333333333, \"cost\": 1.92e-06, \"time\": 0.023}, \"57757ef15e\": {\"quality\": 0.5002333333333333, \"cost\": 2.48e-06, \"time\": 0.0224}, \"579c81bbe0\": {\"quality\": 0.5291, \"cost\": 2.12e-06, \"time\": 0.0165}, \"57bed1722f\": {\"quality\": 0.41585, \"cost\": 1.5599999999999999e-06, \"time\": 0.0138}, \"585ba6d20b\": {\"quality\": 0.5328999999999999, \"cost\": 3.23e-06, \"time\": 0.0322}, \"589267ac64\": {\"quality\": 0.58975, \"cost\": 1.3700000000000001e-05, \"time\": 0.0191}, \"589a1cea79\": {\"quality\": 0.47492500000000004, \"cost\": 2.84e-06, \"time\": 0.025}, \"58ca42839b\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.0164}, \"58dc373441\": {\"quality\": 0.5076999999999999, \"cost\": 1.358e-05, \"time\": 0.0223}, \"59006532b4\": {\"quality\": 0.49876666666666675, \"cost\": 3.15e-06, \"time\": 0.0322}, \"59326c4e00\": {\"quality\": 0.5291, \"cost\": 2.12e-06, \"time\": 0.0165}, \"593975c75b\": {\"quality\": 0.695925, \"cost\": 2.751e-05, \"time\": 0.03950000000000001}, \"596b4f8694\": {\"quality\": 0.50525, \"cost\": 1.4420000000000001e-05, \"time\": 0.0276}, \"5971ba4e0d\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"5996465c0a\": {\"quality\": 0.563225, \"cost\": 1.481e-05, \"time\": 0.0348}, \"59d70b9f65\": {\"quality\": 0.6695333333333333, \"cost\": 1.4e-05, \"time\": 0.0275}, \"59f887b67c\": {\"quality\": 0.47492500000000004, \"cost\": 2.84e-06, \"time\": 0.025}, \"5a22920db4\": {\"quality\": 0.5367, \"cost\": 1.11e-06, \"time\": 0.0157}, \"5a35020d45\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"5aa43da1fc\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.0244}, \"5ae0d88127\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.016399999999999998}, \"5b10fbdbe1\": {\"quality\": 0.5399750000000001, \"cost\": 4.24e-06, \"time\": 0.0363}, \"5bade9eb85\": {\"quality\": 0.523375, \"cost\": 3.06e-06, \"time\": 0.0334}, \"5be16744bf\": {\"quality\": 0.4969666666666666, \"cost\": 3.32e-06, \"time\": 0.0277}, \"5c5055e252\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.0164}, \"5c53feccd9\": {\"quality\": 0.48090000000000005, \"cost\": 4.52e-06, \"time\": 0.038900000000000004}, \"5c77c7c2b2\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"5d298b5b48\": {\"quality\": 0.559425, \"cost\": 1.582e-05, \"time\": 0.0356}, \"5d41515d2e\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"5d4babc723\": {\"quality\": 0.5618749999999999, \"cost\": 1.498e-05, \"time\": 0.0303}, \"5d79b50feb\": {\"quality\": 0.5724666666666667, \"cost\": 3.88e-06, \"time\": 0.030399999999999996}, \"5dc216cd6b\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.030999999999999996}, \"5dd68c1b8f\": {\"quality\": 0.5367, \"cost\": 1.11e-06, \"time\": 0.0157}, \"5de4a882c1\": {\"quality\": 0.6497666666666667, \"cost\": 4.27e-06, \"time\": 0.037599999999999995}, \"5e04e1c72d\": {\"quality\": 0.6453666666666668, \"cost\": 2.536e-05, \"time\": 0.0217}, \"5e923cee9e\": {\"quality\": 0.55235, \"cost\": 1.481e-05, \"time\": 0.0315}, \"5ea2fab380\": {\"quality\": 0.4376, \"cost\": 1.5599999999999999e-06, \"time\": 0.0171}, \"5eb3bb525b\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.022399999999999996}, \"5eea899380\": {\"quality\": 0.6174000000000001, \"cost\": 1.6210000000000002e-05, \"time\": 0.042800000000000005}, \"5f37b3902b\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027699999999999995}, \"5f9282df3c\": {\"quality\": 0.50525, \"cost\": 1.4420000000000001e-05, \"time\": 0.027600000000000003}, \"6019884cf3\": {\"quality\": 0.560775, \"cost\": 1.565e-05, \"time\": 0.040100000000000004}, \"606352363e\": {\"quality\": 0.5028, \"cost\": 1.526e-05, \"time\": 0.0329}, \"608728f868\": {\"quality\": 0.55235, \"cost\": 1.481e-05, \"time\": 0.0315}, \"60b9e936f1\": {\"quality\": 0.5729, \"cost\": 1.286e-05, \"time\": 0.0105}, \"60cb623c53\": {\"quality\": 0.42074999999999996, \"cost\": 7.2e-07, \"time\": 0.0085}, \"6234de86b4\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"62352c6854\": {\"quality\": 0.5678500000000001, \"cost\": 1.666e-05, \"time\": 0.0442}, \"63a0aaebed\": {\"quality\": 0.51495, \"cost\": 1.11e-06, \"time\": 0.0124}, \"647dda686f\": {\"quality\": 0.61985, \"cost\": 1.537e-05, \"time\": 0.0375}, \"6511b21ded\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"652c0f4bdf\": {\"quality\": 0.47247500000000003, \"cost\": 3.68e-06, \"time\": 0.0303}, \"6533c85913\": {\"quality\": 0.47382500000000005, \"cost\": 3.51e-06, \"time\": 0.0348}, \"65627426e0\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"65b76da9c6\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.022399999999999996}, \"65be1c1306\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"65e0216208\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"65eee615d7\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"6623d7a5ac\": {\"quality\": 0.66695, \"cost\": 1.576e-05, \"time\": 0.041400000000000006}, \"66750c0934\": {\"quality\": 0.6403333333333333, \"cost\": 3.26e-06, \"time\": 0.0335}, \"66776ec181\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"66e5ae0a21\": {\"quality\": 0.5406666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.025}, \"6750a8d7a7\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"67632141f6\": {\"quality\": 0.41585, \"cost\": 1.5599999999999999e-06, \"time\": 0.0138}, \"67868fcff6\": {\"quality\": 0.5837, \"cost\": 4.7200000000000005e-06, \"time\": 0.03899999999999999}, \"67bab6732d\": {\"quality\": 0.5082000000000001, \"cost\": 4.16e-06, \"time\": 0.0363}, \"67fe399cf1\": {\"quality\": 0.5147333333333334, \"cost\": 2.48e-06, \"time\": 0.025699999999999997}, \"6846bd8fb3\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"68b4cc3e39\": {\"quality\": 0.581325, \"cost\": 2.6560000000000003e-05, \"time\": 0.0296}, \"69a029ae36\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.024399999999999998}, \"69b3b67de6\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"69bf3f6ba0\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.0224}, \"6a022c3f73\": {\"quality\": 0.42799999999999994, \"cost\": 1.08e-06, \"time\": 0.0144}, \"6a10c53ad8\": {\"quality\": 0.588425, \"cost\": 4.46e-06, \"time\": 0.044700000000000004}, \"6a6348f69d\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"6a74a11bee\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"6a90ec29dd\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.0244}, \"6aac59742a\": {\"quality\": 0.46540000000000004, \"cost\": 2.67e-06, \"time\": 0.0262}, \"6b0862c597\": {\"quality\": 0.6067333333333332, \"cost\": 1.361e-05, \"time\": 0.0236}, \"6b3c16def2\": {\"quality\": 0.5367, \"cost\": 1.11e-06, \"time\": 0.0157}, \"6b99b0e901\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.0244}, \"6b9b2b3515\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"6bcc02962b\": {\"quality\": 0.6884, \"cost\": 1.602e-05, \"time\": 0.035699999999999996}, \"6c1987a9e3\": {\"quality\": 0.57275, \"cost\": 1.498e-05, \"time\": 0.0336}, \"6c50123ee1\": {\"quality\": 0.68885, \"cost\": 1.325e-05, \"time\": 0.0177}, \"6c67c36480\": {\"quality\": 0.55235, \"cost\": 1.481e-05, \"time\": 0.0315}, \"6c9b9f1363\": {\"quality\": 0.610325, \"cost\": 1.52e-05, \"time\": 0.0387}, \"6cc813aa68\": {\"quality\": 0.5291, \"cost\": 2.12e-06, \"time\": 0.0165}, \"6d20c6ace0\": {\"quality\": 0.5724666666666667, \"cost\": 3.88e-06, \"time\": 0.0304}, \"6d444fe21a\": {\"quality\": 0.5406666666666666, \"cost\": 1.4060000000000001e-05, \"time\": 0.025}, \"6db70dc3b6\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"6e0690f576\": {\"quality\": 0.42074999999999996, \"cost\": 7.2e-07, \"time\": 0.0085}, \"6e06cc804f\": {\"quality\": 0.6308666666666666, \"cost\": 2.536e-05, \"time\": 0.0184}, \"6e24048a2e\": {\"quality\": 0.5406666666666666, \"cost\": 1.4060000000000001e-05, \"time\": 0.025}, \"6e93514f45\": {\"quality\": 0.4102333333333333, \"cost\": 1.92e-06, \"time\": 0.016399999999999998}, \"6eae47102b\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.0263}, \"6ed4cae469\": {\"quality\": 0.7468, \"cost\": 1.25e-05, \"time\": 0.0079}, \"6ef3b7127e\": {\"quality\": 0.6592, \"cost\": 3.52e-06, \"time\": 0.0278}, \"6f60a05c33\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"6fe0b3f929\": {\"quality\": 0.42146666666666666, \"cost\": 2.76e-06, \"time\": 0.025}, \"700ab1d309\": {\"quality\": 0.48573333333333335, \"cost\": 2.48e-06, \"time\": 0.0191}, \"7040e83d52\": {\"quality\": 0.6497666666666667, \"cost\": 4.27e-06, \"time\": 0.037599999999999995}, \"7046765af8\": {\"quality\": 0.55235, \"cost\": 1.481e-05, \"time\": 0.0315}, \"70b7c92ce8\": {\"quality\": 0.5656749999999999, \"cost\": 1.397e-05, \"time\": 0.0295}, \"70c850e039\": {\"quality\": 0.5318, \"cost\": 1.95e-06, \"time\": 0.020999999999999998}, \"7112a7e64c\": {\"quality\": 0.64505, \"cost\": 2.51e-06, \"time\": 0.0237}, \"71b615468b\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027700000000000002}, \"723fd5589a\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"7250da0f41\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027700000000000002}, \"7274a50778\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"72d022ce33\": {\"quality\": 0.6393, \"cost\": 2.695e-05, \"time\": 0.0368}, \"7347cf0308\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"736e652158\": {\"quality\": 0.46785, \"cost\": 1.8299999999999998e-06, \"time\": 0.020900000000000002}, \"739b1f81dc\": {\"quality\": 0.5367, \"cost\": 1.11e-06, \"time\": 0.0157}, \"742a5c0552\": {\"quality\": 0.7030000000000001, \"cost\": 1.426e-05, \"time\": 0.0218}, \"742ec1b2e1\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.028900000000000002}, \"7435fd54f8\": {\"quality\": 0.5656749999999999, \"cost\": 1.397e-05, \"time\": 0.0295}, \"7445d99939\": {\"quality\": 0.43596666666666667, \"cost\": 2.76e-06, \"time\": 0.0283}, \"7466a5f424\": {\"quality\": 0.537525, \"cost\": 5.0800000000000005e-06, \"time\": 0.0416}, \"74a0be215b\": {\"quality\": 0.5161250000000001, \"cost\": 1.4420000000000001e-05, \"time\": 0.0309}, \"74d7f64b8c\": {\"quality\": 0.46785, \"cost\": 1.8299999999999998e-06, \"time\": 0.020900000000000002}, \"7524905580\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"752d9649f2\": {\"quality\": 0.5484, \"cost\": 5.0800000000000005e-06, \"time\": 0.044899999999999995}, \"75d61c2cd0\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.028900000000000002}, \"7604c0aa13\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027699999999999995}, \"76c09db721\": {\"quality\": 0.61985, \"cost\": 1.537e-05, \"time\": 0.0375}, \"774f268b66\": {\"quality\": 0.6417499999999999, \"cost\": 2.611e-05, \"time\": 0.0315}, \"7765576286\": {\"quality\": 0.4847, \"cost\": 3.51e-06, \"time\": 0.0381}, \"77c02b00c1\": {\"quality\": 0.48335, \"cost\": 3.68e-06, \"time\": 0.033600000000000005}, \"7801da66b9\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"782d52674e\": {\"quality\": 0.55235, \"cost\": 1.481e-05, \"time\": 0.0315}, \"786e5d0af5\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"7878563d63\": {\"quality\": 0.7030000000000001, \"cost\": 1.426e-05, \"time\": 0.0218}, \"79e1ca9b3c\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.028900000000000002}, \"79fad58f07\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"7a207b42a8\": {\"quality\": 0.4183, \"cost\": 2.2799999999999998e-06, \"time\": 0.0223}, \"7a2cdc546c\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.0289}, \"7a42a77788\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.0217}, \"7a58d3472b\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"7b3937c1f1\": {\"quality\": 0.7030000000000001, \"cost\": 1.426e-05, \"time\": 0.0218}, \"7b6f44618e\": {\"quality\": 0.4917750000000001, \"cost\": 4.52e-06, \"time\": 0.0422}, \"7c62576527\": {\"quality\": 0.561875, \"cost\": 1.498e-05, \"time\": 0.0303}, \"7c89a2b69e\": {\"quality\": 0.53045, \"cost\": 4.07e-06, \"time\": 0.0375}, \"7c96c9712f\": {\"quality\": 0.58975, \"cost\": 1.3700000000000001e-05, \"time\": 0.0191}, \"7ca066aa1c\": {\"quality\": 0.5548, \"cost\": 1.397e-05, \"time\": 0.0262}, \"7cb5591f27\": {\"quality\": 0.6421, \"cost\": 2.62e-05, \"time\": 0.027}, \"7cf56a7fbc\": {\"quality\": 0.6129000000000001, \"cost\": 1.546e-05, \"time\": 0.033}, \"7d44f0959d\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"7d60c38c5c\": {\"quality\": 0.6309, \"cost\": 1.5e-06, \"time\": 0.0196}, \"7d9b4535ac\": {\"quality\": 0.612775, \"cost\": 1.436e-05, \"time\": 0.0334}, \"7daf7ff182\": {\"quality\": 0.5082000000000001, \"cost\": 4.16e-06, \"time\": 0.0363}, \"7dcedb3d02\": {\"quality\": 0.61985, \"cost\": 1.537e-05, \"time\": 0.037500000000000006}, \"7e22f12cd1\": {\"quality\": 0.5548, \"cost\": 1.397e-05, \"time\": 0.0262}, \"7e53a50b13\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"7ed07ad40a\": {\"quality\": 0.5648333333333334, \"cost\": 2.7e-06, \"time\": 0.030799999999999998}, \"7fa67a7656\": {\"quality\": 0.5114666666666666, \"cost\": 3.32e-06, \"time\": 0.031}, \"7fc6c84bdf\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"7ff8a779cc\": {\"quality\": 0.6067333333333332, \"cost\": 1.361e-05, \"time\": 0.0236}, \"801af99400\": {\"quality\": 0.6695333333333333, \"cost\": 1.4e-05, \"time\": 0.0275}, \"806881adcb\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"80ad4122e4\": {\"quality\": 0.5618749999999999, \"cost\": 1.498e-05, \"time\": 0.0303}, \"80be7df955\": {\"quality\": 0.42074999999999996, \"cost\": 7.2e-07, \"time\": 0.0085}, \"80bf60c422\": {\"quality\": 0.51495, \"cost\": 1.11e-06, \"time\": 0.0124}, \"81333c7a33\": {\"quality\": 0.6592, \"cost\": 5.28e-06, \"time\": 0.0417}, \"813e75210b\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"81660ae8b2\": {\"quality\": 0.494375, \"cost\": 1.4420000000000001e-05, \"time\": 0.024300000000000002}, \"816958b5d1\": {\"quality\": 0.608975, \"cost\": 1.537e-05, \"time\": 0.034199999999999994}, \"81ab2ef3f4\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.0164}, \"829df73946\": {\"quality\": 0.64505, \"cost\": 2.51e-06, \"time\": 0.0237}, \"82ea1bd1b9\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"8357183895\": {\"quality\": 0.46540000000000004, \"cost\": 2.67e-06, \"time\": 0.0262}, \"8392a6083a\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.033600000000000005}, \"83aee532b7\": {\"quality\": 0.543925, \"cost\": 1.397e-05, \"time\": 0.022899999999999997}, \"83b26646c3\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"83c9e66ec6\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"847a0e5db5\": {\"quality\": 0.6129, \"cost\": 1.546e-05, \"time\": 0.033}, \"847fd49235\": {\"quality\": 0.42799999999999994, \"cost\": 1.08e-06, \"time\": 0.0144}, \"849100224d\": {\"quality\": 0.5837749999999999, \"cost\": 2.572e-05, \"time\": 0.024300000000000002}, \"84b91c37ab\": {\"quality\": 0.608975, \"cost\": 1.537e-05, \"time\": 0.034199999999999994}, \"8519bef585\": {\"quality\": 0.42074999999999996, \"cost\": 7.2e-07, \"time\": 0.0085}, \"85c94a5505\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"85eda38404\": {\"quality\": 0.58975, \"cost\": 1.3700000000000001e-05, \"time\": 0.0191}, \"862183bfb9\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"8631e49c94\": {\"quality\": 0.5548, \"cost\": 1.397e-05, \"time\": 0.026199999999999998}, \"8668f65f05\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"86bf6375af\": {\"quality\": 0.560775, \"cost\": 1.565e-05, \"time\": 0.040100000000000004}, \"870e2f87b4\": {\"quality\": 0.587075, \"cost\": 4.63e-06, \"time\": 0.0402}, \"887ad124e1\": {\"quality\": 0.5053, \"cost\": 1.47e-06, \"time\": 0.021599999999999998}, \"8886cb3082\": {\"quality\": 0.5328999999999999, \"cost\": 3.23e-06, \"time\": 0.0322}, \"88e71efa9b\": {\"quality\": 0.5594250000000001, \"cost\": 1.582e-05, \"time\": 0.03560000000000001}, \"8941621423\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"8950a6efe0\": {\"quality\": 0.5439333333333334, \"cost\": 1.322e-05, \"time\": 0.0197}, \"8961e4d901\": {\"quality\": 0.4102333333333333, \"cost\": 1.92e-06, \"time\": 0.016399999999999998}, \"8974aa89a0\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"89836d2020\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"89a289907e\": {\"quality\": 0.590875, \"cost\": 3.62e-06, \"time\": 0.0394}, \"89a35a09b1\": {\"quality\": 0.48714999999999997, \"cost\": 2.67e-06, \"time\": 0.0328}, \"8a3a35c762\": {\"quality\": 0.626925, \"cost\": 1.6380000000000002e-05, \"time\": 0.0416}, \"8a50695d1f\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.0217}, \"8ab351aa13\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.024399999999999998}, \"8ac8b5773a\": {\"quality\": 0.563225, \"cost\": 1.481e-05, \"time\": 0.034800000000000005}, \"8acd758b7f\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"8b10891ea5\": {\"quality\": 0.5678500000000001, \"cost\": 1.666e-05, \"time\": 0.0442}, \"8b721bbc6f\": {\"quality\": 0.48714999999999997, \"cost\": 2.67e-06, \"time\": 0.0328}, \"8b77535cce\": {\"quality\": 0.6174, \"cost\": 1.6210000000000002e-05, \"time\": 0.042800000000000005}, \"8bbbe0f52a\": {\"quality\": 0.4392333333333333, \"cost\": 1.92e-06, \"time\": 0.023}, \"8bc184f385\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.026899999999999997}, \"8bf5c3eadc\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"8bf80a50cb\": {\"quality\": 0.61985, \"cost\": 1.537e-05, \"time\": 0.0375}, \"8c274ca255\": {\"quality\": 0.6740250000000001, \"cost\": 1.677e-05, \"time\": 0.0455}, \"8d7594020b\": {\"quality\": 0.5406666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.025}, \"8d79e03266\": {\"quality\": 0.6497666666666667, \"cost\": 4.27e-06, \"time\": 0.037599999999999995}, \"8d90814b94\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"8e1a01da19\": {\"quality\": 0.5548, \"cost\": 1.397e-05, \"time\": 0.0262}, \"8e2498635d\": {\"quality\": 0.7176, \"cost\": 2.676e-05, \"time\": 0.0297}, \"8e5842ccbd\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.026899999999999997}, \"8e5daf241e\": {\"quality\": 0.6695333333333333, \"cost\": 1.4e-05, \"time\": 0.0275}, \"8e9715ee01\": {\"quality\": 0.6453666666666668, \"cost\": 2.536e-05, \"time\": 0.0217}, \"8e9b7300d4\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"8f29fab8ac\": {\"quality\": 0.513675, \"cost\": 1.526e-05, \"time\": 0.0362}, \"8f44d89429\": {\"quality\": 0.6884, \"cost\": 1.602e-05, \"time\": 0.035699999999999996}, \"8f4caddfe6\": {\"quality\": 0.5869666666666667, \"cost\": 3.88e-06, \"time\": 0.033699999999999994}, \"8f4edde3f0\": {\"quality\": 0.4857333333333333, \"cost\": 2.48e-06, \"time\": 0.0191}, \"8f9cefbc22\": {\"quality\": 0.7030000000000001, \"cost\": 1.426e-05, \"time\": 0.0218}, \"9025e2480f\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"9028588af4\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"9059fd80ad\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"90d5e40c1b\": {\"quality\": 0.42146666666666666, \"cost\": 2.76e-06, \"time\": 0.025}, \"90d9a86a2a\": {\"quality\": 0.49876666666666675, \"cost\": 3.15e-06, \"time\": 0.0322}, \"90ff13783c\": {\"quality\": 0.5304500000000001, \"cost\": 4.07e-06, \"time\": 0.0375}, \"90ff8eb055\": {\"quality\": 0.426725, \"cost\": 3.1199999999999998e-06, \"time\": 0.030900000000000004}, \"9104e31369\": {\"quality\": 0.53045, \"cost\": 4.07e-06, \"time\": 0.0375}, \"918983323f\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"91928dfdd9\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.028900000000000002}, \"91c800af6b\": {\"quality\": 0.47382500000000005, \"cost\": 3.51e-06, \"time\": 0.0348}, \"91e841cfd5\": {\"quality\": 0.61985, \"cost\": 1.537e-05, \"time\": 0.037500000000000006}, \"9253901a1f\": {\"quality\": 0.5594250000000001, \"cost\": 1.582e-05, \"time\": 0.0356}, \"9288642e53\": {\"quality\": 0.5291, \"cost\": 2.12e-06, \"time\": 0.0165}, \"92ba9c5be3\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"92c9dcd43b\": {\"quality\": 0.543775, \"cost\": 3.23e-06, \"time\": 0.0355}, \"93011c0821\": {\"quality\": 0.53045, \"cost\": 4.07e-06, \"time\": 0.0375}, \"933b4d17dd\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"9373267bdb\": {\"quality\": 0.61985, \"cost\": 1.537e-05, \"time\": 0.0375}, \"94010928c6\": {\"quality\": 0.42799999999999994, \"cost\": 1.08e-06, \"time\": 0.0144}, \"9403809e44\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"940c88ddc5\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.0164}, \"948f4081ba\": {\"quality\": 0.563225, \"cost\": 1.481e-05, \"time\": 0.0348}, \"94ac356663\": {\"quality\": 0.5114666666666666, \"cost\": 3.32e-06, \"time\": 0.031}, \"94dff9a424\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"9508356a2e\": {\"quality\": 0.4763, \"cost\": 1.47e-06, \"time\": 0.015}, \"9539d0e28c\": {\"quality\": 0.561875, \"cost\": 1.498e-05, \"time\": 0.0303}, \"956bdcc254\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.0224}, \"957d0dafc5\": {\"quality\": 0.6884, \"cost\": 1.602e-05, \"time\": 0.035699999999999996}, \"9594b0c783\": {\"quality\": 0.5053, \"cost\": 1.47e-06, \"time\": 0.021599999999999998}, \"95a7b80c2a\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.0289}, \"964c671f18\": {\"quality\": 0.587075, \"cost\": 4.63e-06, \"time\": 0.0402}, \"9679fe2b69\": {\"quality\": 0.5053, \"cost\": 1.47e-06, \"time\": 0.021599999999999998}, \"968fc95038\": {\"quality\": 0.5053, \"cost\": 1.47e-06, \"time\": 0.0216}, \"96b487c724\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"96c30205f5\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.024399999999999998}, \"96f87d6483\": {\"quality\": 0.53045, \"cost\": 4.07e-06, \"time\": 0.0375}, \"972c83b002\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.023599999999999996}, \"977a4d6b6b\": {\"quality\": 0.5484, \"cost\": 5.0800000000000005e-06, \"time\": 0.044899999999999995}, \"97bc30bd83\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"980db5f95f\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.0164}, \"9836765d41\": {\"quality\": 0.5406666666666666, \"cost\": 1.4060000000000001e-05, \"time\": 0.024999999999999998}, \"99569e3937\": {\"quality\": 0.6174, \"cost\": 1.6210000000000002e-05, \"time\": 0.042800000000000005}, \"99ea16a9a6\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"9a5b39370f\": {\"quality\": 0.5509999999999999, \"cost\": 1.498e-05, \"time\": 0.026999999999999996}, \"9aa4abfb50\": {\"quality\": 0.39899999999999997, \"cost\": 7.2e-07, \"time\": 0.0052}, \"9ad7a98c31\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"9b3fb79bcb\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"9b6d4915f3\": {\"quality\": 0.43270000000000003, \"cost\": 3.6e-06, \"time\": 0.0336}, \"9bae5bafc1\": {\"quality\": 0.5869666666666667, \"cost\": 3.88e-06, \"time\": 0.033699999999999994}, \"9be8a5f317\": {\"quality\": 0.5703, \"cost\": 1.582e-05, \"time\": 0.038900000000000004}, \"9c549db0a7\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"9c85f8cfcb\": {\"quality\": 0.51495, \"cost\": 1.11e-06, \"time\": 0.0124}, \"9c8cc46e6c\": {\"quality\": 0.51495, \"cost\": 1.11e-06, \"time\": 0.0124}, \"9c97d35a30\": {\"quality\": 0.5680999999999999, \"cost\": 1.86e-06, \"time\": 0.0255}, \"9cbe7858a2\": {\"quality\": 0.50525, \"cost\": 1.4420000000000001e-05, \"time\": 0.0276}, \"9ce2c3fd98\": {\"quality\": 0.4135, \"cost\": 1.08e-06, \"time\": 0.0111}, \"9d18cd0737\": {\"quality\": 0.5318, \"cost\": 1.95e-06, \"time\": 0.020999999999999998}, \"9e06360bc9\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"9f07a95e69\": {\"quality\": 0.61605, \"cost\": 1.6380000000000002e-05, \"time\": 0.0383}, \"9fb157be35\": {\"quality\": 0.48573333333333335, \"cost\": 2.48e-06, \"time\": 0.0191}, \"a04ac8e33a\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.0217}, \"a04bc6e116\": {\"quality\": 0.5406666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.025}, \"a0b81be5b4\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"a0c85d260e\": {\"quality\": 0.5955, \"cost\": 5.47e-06, \"time\": 0.048799999999999996}, \"a0dc9f50ac\": {\"quality\": 0.43596666666666667, \"cost\": 2.76e-06, \"time\": 0.0283}, \"a18225b7b5\": {\"quality\": 0.6884, \"cost\": 1.602e-05, \"time\": 0.035699999999999996}, \"a1d822289e\": {\"quality\": 0.6421, \"cost\": 2.62e-05, \"time\": 0.027}, \"a25596c056\": {\"quality\": 0.513675, \"cost\": 1.526e-05, \"time\": 0.0362}, \"a2811c7324\": {\"quality\": 0.6067333333333332, \"cost\": 1.361e-05, \"time\": 0.0236}, \"a2aa082d14\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027700000000000002}, \"a2cd339ad9\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.040799999999999996}, \"a2fd03e6a5\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"a31e87d7cb\": {\"quality\": 0.4425, \"cost\": 7.2e-07, \"time\": 0.0118}, \"a3e23c327b\": {\"quality\": 0.561875, \"cost\": 1.498e-05, \"time\": 0.030299999999999997}, \"a457f6c300\": {\"quality\": 0.4135, \"cost\": 1.08e-06, \"time\": 0.0111}, \"a47de025c8\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"a515a9c8cc\": {\"quality\": 0.4376, \"cost\": 1.5599999999999999e-06, \"time\": 0.0171}, \"a5949b76ec\": {\"quality\": 0.42146666666666666, \"cost\": 2.76e-06, \"time\": 0.025}, \"a60dd076b8\": {\"quality\": 0.4392333333333333, \"cost\": 1.92e-06, \"time\": 0.023}, \"a6297a6c56\": {\"quality\": 0.4376, \"cost\": 1.5599999999999999e-06, \"time\": 0.0171}, \"a62b7555b9\": {\"quality\": 0.5703, \"cost\": 1.582e-05, \"time\": 0.0389}, \"a63f48e8ca\": {\"quality\": 0.5406666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.025}, \"a6460dbb7c\": {\"quality\": 0.64505, \"cost\": 2.51e-06, \"time\": 0.0237}, \"a66a4cf4b0\": {\"quality\": 0.61605, \"cost\": 1.6380000000000002e-05, \"time\": 0.0383}, \"a6e2d69222\": {\"quality\": 0.6129000000000001, \"cost\": 1.546e-05, \"time\": 0.033}, \"a717c4c535\": {\"quality\": 0.4917750000000001, \"cost\": 4.52e-06, \"time\": 0.0422}, \"a76afe9960\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"a7a6353090\": {\"quality\": 0.6067333333333332, \"cost\": 1.361e-05, \"time\": 0.0236}, \"a80f6535b1\": {\"quality\": 0.59465, \"cost\": 1.286e-05, \"time\": 0.0138}, \"a86b137d7f\": {\"quality\": 0.5508500000000001, \"cost\": 2.12e-06, \"time\": 0.019799999999999998}, \"a88eb1493c\": {\"quality\": 0.5318, \"cost\": 1.95e-06, \"time\": 0.020999999999999998}, \"a89c533d6c\": {\"quality\": 0.5374, \"cost\": 1.4900000000000001e-05, \"time\": 0.0303}, \"a8d8264600\": {\"quality\": 0.6174000000000001, \"cost\": 1.621e-05, \"time\": 0.042800000000000005}, \"a8eb36b210\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.0289}, \"a95b4a6dd0\": {\"quality\": 0.42146666666666666, \"cost\": 2.76e-06, \"time\": 0.025}, \"a9621ea4e6\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.0197}, \"a9721a0a50\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.030999999999999996}, \"a972b02c61\": {\"quality\": 0.7468, \"cost\": 1.25e-05, \"time\": 0.0079}, \"a9c5c4e311\": {\"quality\": 0.6067333333333332, \"cost\": 1.361e-05, \"time\": 0.0236}, \"a9d96670eb\": {\"quality\": 0.5439333333333334, \"cost\": 1.322e-05, \"time\": 0.0197}, \"a9e8c974d3\": {\"quality\": 0.5509999999999999, \"cost\": 1.498e-05, \"time\": 0.026999999999999996}, \"aa08180e36\": {\"quality\": 0.4858, \"cost\": 2.84e-06, \"time\": 0.0283}, \"aa38702a02\": {\"quality\": 0.42799999999999994, \"cost\": 1.08e-06, \"time\": 0.0144}, \"aadbfc418b\": {\"quality\": 0.44250000000000006, \"cost\": 1.08e-06, \"time\": 0.0177}, \"aaeb8b0010\": {\"quality\": 0.50525, \"cost\": 1.4420000000000001e-05, \"time\": 0.0276}, \"ab288ee7f2\": {\"quality\": 0.563225, \"cost\": 1.481e-05, \"time\": 0.0348}, \"ab43b02cb0\": {\"quality\": 0.5082000000000001, \"cost\": 4.16e-06, \"time\": 0.0363}, \"aba1d612cc\": {\"quality\": 0.648825, \"cost\": 2.712e-05, \"time\": 0.0356}, \"ac208e7a1d\": {\"quality\": 0.5724666666666667, \"cost\": 3.88e-06, \"time\": 0.0304}, \"ac2224adbe\": {\"quality\": 0.6129000000000001, \"cost\": 1.546e-05, \"time\": 0.033}, \"ac828ffe70\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"ac9fdc1550\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"aca957ecff\": {\"quality\": 0.6417499999999999, \"cost\": 2.611e-05, \"time\": 0.0315}, \"acfe1ed920\": {\"quality\": 0.59465, \"cost\": 1.286e-05, \"time\": 0.0138}, \"ad3efe44c3\": {\"quality\": 0.39899999999999997, \"cost\": 7.2e-07, \"time\": 0.0052}, \"ad41c95a99\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"ad48432c22\": {\"quality\": 0.46540000000000004, \"cost\": 2.67e-06, \"time\": 0.0262}, \"ad6ebbba8d\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"ad90055ef6\": {\"quality\": 0.39899999999999997, \"cost\": 7.2e-07, \"time\": 0.0052}, \"adab1e0fb1\": {\"quality\": 0.5508500000000001, \"cost\": 2.12e-06, \"time\": 0.019799999999999998}, \"ae655ec593\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"ae94b172be\": {\"quality\": 0.5002333333333333, \"cost\": 2.48e-06, \"time\": 0.0224}, \"aec9dc5873\": {\"quality\": 0.5729, \"cost\": 1.286e-05, \"time\": 0.0105}, \"af360c323c\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.018299999999999997}, \"af90567194\": {\"quality\": 0.587075, \"cost\": 4.63e-06, \"time\": 0.0402}, \"afe77d0f89\": {\"quality\": 0.66695, \"cost\": 1.576e-05, \"time\": 0.041400000000000006}, \"b0948c05b6\": {\"quality\": 0.5837, \"cost\": 4.7200000000000005e-06, \"time\": 0.03899999999999999}, \"b0c4a6640b\": {\"quality\": 0.5548, \"cost\": 1.397e-05, \"time\": 0.0262}, \"b12caafd58\": {\"quality\": 0.5076999999999999, \"cost\": 1.358e-05, \"time\": 0.0223}, \"b18168b9c1\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"b1a7428a01\": {\"quality\": 0.5703, \"cost\": 1.582e-05, \"time\": 0.038900000000000004}, \"b1acdebb48\": {\"quality\": 0.6884, \"cost\": 1.602e-05, \"time\": 0.035699999999999996}, \"b1b06f4ee7\": {\"quality\": 0.6421, \"cost\": 2.62e-05, \"time\": 0.027000000000000003}, \"b1cf8d33e5\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"b1e9ab6b1a\": {\"quality\": 0.559425, \"cost\": 1.582e-05, \"time\": 0.0356}, \"b2b057ba41\": {\"quality\": 0.5680999999999999, \"cost\": 1.86e-06, \"time\": 0.0255}, \"b2e063499d\": {\"quality\": 0.6497666666666667, \"cost\": 4.27e-06, \"time\": 0.037599999999999995}, \"b33412410e\": {\"quality\": 0.5374, \"cost\": 1.4900000000000001e-05, \"time\": 0.0303}, \"b3369775dc\": {\"quality\": 0.588425, \"cost\": 4.46e-06, \"time\": 0.044700000000000004}, \"b3b9205f60\": {\"quality\": 0.50525, \"cost\": 1.4420000000000001e-05, \"time\": 0.027600000000000003}, \"b3c56f0b3c\": {\"quality\": 0.6421, \"cost\": 2.62e-05, \"time\": 0.027}, \"b3f20b706d\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"b4002173ee\": {\"quality\": 0.43270000000000003, \"cost\": 2.4e-06, \"time\": 0.0224}, \"b46d382384\": {\"quality\": 0.6453666666666668, \"cost\": 2.536e-05, \"time\": 0.0217}, \"b4b2482ef9\": {\"quality\": 0.6129000000000001, \"cost\": 1.546e-05, \"time\": 0.033}, \"b4be043238\": {\"quality\": 0.6308666666666666, \"cost\": 2.536e-05, \"time\": 0.0184}, \"b531bd0548\": {\"quality\": 0.581325, \"cost\": 2.6560000000000003e-05, \"time\": 0.0296}, \"b56c312eda\": {\"quality\": 0.4847, \"cost\": 3.51e-06, \"time\": 0.0381}, \"b5e2b41c1c\": {\"quality\": 0.5536, \"cost\": 1.86e-06, \"time\": 0.022199999999999998}, \"b61ce57a90\": {\"quality\": 0.5703, \"cost\": 1.582e-05, \"time\": 0.038900000000000004}, \"b64ddb14f9\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"b67107a43e\": {\"quality\": 0.46785, \"cost\": 1.8299999999999998e-06, \"time\": 0.020900000000000002}, \"b67720aa5c\": {\"quality\": 0.5149333333333334, \"cost\": 1.322e-05, \"time\": 0.0131}, \"b682a23b89\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"b69ef5add4\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.0164}, \"b796b7ffd3\": {\"quality\": 0.590875, \"cost\": 3.62e-06, \"time\": 0.0394}, \"b7a0083dc4\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"b7d0e8557f\": {\"quality\": 0.48714999999999997, \"cost\": 2.67e-06, \"time\": 0.0328}, \"b8317a3a8c\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"b8ab3d2f25\": {\"quality\": 0.5869666666666667, \"cost\": 3.88e-06, \"time\": 0.033699999999999994}, \"b8b569172f\": {\"quality\": 0.4969666666666666, \"cost\": 3.32e-06, \"time\": 0.0277}, \"b8f5ab44bb\": {\"quality\": 0.6244750000000001, \"cost\": 1.722e-05, \"time\": 0.0469}, \"b91e7fdb29\": {\"quality\": 0.4183, \"cost\": 2.2799999999999998e-06, \"time\": 0.0223}, \"b932beaaa6\": {\"quality\": 0.6129, \"cost\": 1.546e-05, \"time\": 0.033}, \"b9770c2261\": {\"quality\": 0.6309, \"cost\": 1.5e-06, \"time\": 0.0196}, \"b9bb1e6f8d\": {\"quality\": 0.588425, \"cost\": 4.46e-06, \"time\": 0.044700000000000004}, \"b9d0e8740c\": {\"quality\": 0.5703, \"cost\": 1.582e-05, \"time\": 0.038900000000000004}, \"b9da208432\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"ba3223f6ac\": {\"quality\": 0.428, \"cost\": 1.08e-06, \"time\": 0.0144}, \"bb13365175\": {\"quality\": 0.608975, \"cost\": 1.537e-05, \"time\": 0.034199999999999994}, \"bb1b3a4d29\": {\"quality\": 0.6019, \"cost\": 1.436e-05, \"time\": 0.0301}, \"bb6536b0ab\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.030999999999999996}, \"bbba9dd6ae\": {\"quality\": 0.64505, \"cost\": 2.51e-06, \"time\": 0.0237}, \"bbde69a1ae\": {\"quality\": 0.5028, \"cost\": 1.526e-05, \"time\": 0.0329}, \"bc29a0c0fe\": {\"quality\": 0.6453666666666668, \"cost\": 2.536e-05, \"time\": 0.0217}, \"bc3d02f753\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"bc4c1fcc64\": {\"quality\": 0.64505, \"cost\": 2.51e-06, \"time\": 0.0237}, \"bd30d27f62\": {\"quality\": 0.6308666666666666, \"cost\": 2.536e-05, \"time\": 0.0184}, \"bd99b2fb21\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.031}, \"bddc7d2a34\": {\"quality\": 0.6019, \"cost\": 1.436e-05, \"time\": 0.0301}, \"be2ae88f70\": {\"quality\": 0.4135, \"cost\": 1.08e-06, \"time\": 0.0111}, \"be4740f38f\": {\"quality\": 0.5837, \"cost\": 4.7200000000000005e-06, \"time\": 0.03899999999999999}, \"bec0c6a95f\": {\"quality\": 0.581325, \"cost\": 2.6560000000000003e-05, \"time\": 0.0296}, \"bed888d4dc\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"bf45e407f6\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"bf5550f320\": {\"quality\": 0.5703, \"cost\": 1.582e-05, \"time\": 0.0389}, \"bf87e58322\": {\"quality\": 0.6695333333333333, \"cost\": 1.4e-05, \"time\": 0.0275}, \"bfed7670ed\": {\"quality\": 0.4969666666666666, \"cost\": 3.32e-06, \"time\": 0.0277}, \"c0541e2220\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.0217}, \"c0e10c0048\": {\"quality\": 0.50525, \"cost\": 1.4420000000000001e-05, \"time\": 0.0276}, \"c127509a7a\": {\"quality\": 0.6403333333333333, \"cost\": 3.26e-06, \"time\": 0.0335}, \"c13682c7c7\": {\"quality\": 0.494225, \"cost\": 3.68e-06, \"time\": 0.0369}, \"c13d6e78e9\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"c14ff3144d\": {\"quality\": 0.4425, \"cost\": 7.2e-07, \"time\": 0.0118}, \"c1e42ac47b\": {\"quality\": 0.6244750000000001, \"cost\": 1.722e-05, \"time\": 0.0469}, \"c2949aa902\": {\"quality\": 0.5648333333333334, \"cost\": 2.7e-06, \"time\": 0.030799999999999998}, \"c31e956b35\": {\"quality\": 0.523375, \"cost\": 3.06e-06, \"time\": 0.0334}, \"c36b525dde\": {\"quality\": 0.5648333333333334, \"cost\": 2.7e-06, \"time\": 0.030799999999999998}, \"c38326e2bd\": {\"quality\": 0.626925, \"cost\": 1.6380000000000002e-05, \"time\": 0.0416}, \"c3ec2cec59\": {\"quality\": 0.5724666666666667, \"cost\": 3.88e-06, \"time\": 0.0304}, \"c44720575f\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"c48ecefab6\": {\"quality\": 0.49682499999999996, \"cost\": 1.358e-05, \"time\": 0.019000000000000003}, \"c4a64eb40f\": {\"quality\": 0.7176, \"cost\": 2.676e-05, \"time\": 0.0297}, \"c4a80d19b3\": {\"quality\": 0.5800000000000001, \"cost\": 3.62e-06, \"time\": 0.03609999999999999}, \"c4c2826afd\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"c4c94a5527\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"c4e75ee9ba\": {\"quality\": 0.7176, \"cost\": 2.676e-05, \"time\": 0.0297}, \"c4f3e7665d\": {\"quality\": 0.4135, \"cost\": 1.08e-06, \"time\": 0.0111}, \"c5471bef57\": {\"quality\": 0.494375, \"cost\": 1.4420000000000001e-05, \"time\": 0.024300000000000002}, \"c54a408db7\": {\"quality\": 0.6067333333333333, \"cost\": 1.361e-05, \"time\": 0.0236}, \"c59cc41335\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"c5a0b065e0\": {\"quality\": 0.5729, \"cost\": 1.286e-05, \"time\": 0.0105}, \"c5a16b834a\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"c5fbe2076f\": {\"quality\": 0.537525, \"cost\": 5.0800000000000005e-06, \"time\": 0.0416}, \"c617370f6b\": {\"quality\": 0.5439333333333334, \"cost\": 1.322e-05, \"time\": 0.0197}, \"c67f782c7f\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.024399999999999998}, \"c691a29c42\": {\"quality\": 0.48090000000000005, \"cost\": 4.52e-06, \"time\": 0.038900000000000004}, \"c6a339987c\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"c772ff3704\": {\"quality\": 0.429175, \"cost\": 2.2799999999999998e-06, \"time\": 0.0256}, \"c7e3f348c2\": {\"quality\": 0.61605, \"cost\": 1.6380000000000002e-05, \"time\": 0.0383}, \"c823589ab6\": {\"quality\": 0.5149333333333334, \"cost\": 1.322e-05, \"time\": 0.0131}, \"c82f834e85\": {\"quality\": 0.5775333333333333, \"cost\": 2.87e-06, \"time\": 0.029599999999999998}, \"c85099881f\": {\"quality\": 0.5729, \"cost\": 1.286e-05, \"time\": 0.0105}, \"c935a33384\": {\"quality\": 0.5484, \"cost\": 5.0800000000000005e-06, \"time\": 0.044899999999999995}, \"ca3177461f\": {\"quality\": 0.50525, \"cost\": 1.4420000000000001e-05, \"time\": 0.0276}, \"caa7c0bd6b\": {\"quality\": 0.49876666666666675, \"cost\": 3.15e-06, \"time\": 0.0322}, \"cac6b051e9\": {\"quality\": 0.53425, \"cost\": 3.06e-06, \"time\": 0.036699999999999997}, \"cacb342f64\": {\"quality\": 0.63795, \"cost\": 2.712e-05, \"time\": 0.032299999999999995}, \"cb9948679c\": {\"quality\": 0.5076999999999999, \"cost\": 1.358e-05, \"time\": 0.0223}, \"cbb5eb0e74\": {\"quality\": 0.53425, \"cost\": 3.06e-06, \"time\": 0.036699999999999997}, \"cbc32cbeff\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.029500000000000002}, \"cbd4461293\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"cbe2318045\": {\"quality\": 0.5147333333333334, \"cost\": 2.48e-06, \"time\": 0.025699999999999997}, \"cc886fe337\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"cc9a6248a0\": {\"quality\": 0.7468, \"cost\": 1.25e-05, \"time\": 0.0079}, \"ccb2335b3f\": {\"quality\": 0.55235, \"cost\": 1.481e-05, \"time\": 0.0315}, \"ccdf03a55b\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.0164}, \"ccf72745c1\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027700000000000002}, \"cd1d418732\": {\"quality\": 0.5082000000000001, \"cost\": 4.16e-06, \"time\": 0.0363}, \"cd23c79db1\": {\"quality\": 0.48573333333333335, \"cost\": 2.48e-06, \"time\": 0.0191}, \"cd64fbfcd9\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.023599999999999996}, \"cd85a01e81\": {\"quality\": 0.55235, \"cost\": 1.481e-05, \"time\": 0.0315}, \"ce4bc5f348\": {\"quality\": 0.42074999999999996, \"cost\": 7.2e-07, \"time\": 0.0085}, \"ce980cf86f\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"ceae8b8bb9\": {\"quality\": 0.608975, \"cost\": 1.537e-05, \"time\": 0.034199999999999994}, \"cecca90dd2\": {\"quality\": 0.5536, \"cost\": 1.86e-06, \"time\": 0.022199999999999998}, \"cf9538faf0\": {\"quality\": 0.5147333333333334, \"cost\": 2.48e-06, \"time\": 0.0257}, \"cf9d2e224c\": {\"quality\": 0.5508500000000001, \"cost\": 2.12e-06, \"time\": 0.019799999999999998}, \"cfd36f3a8c\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.0263}, \"cffa29a6ef\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"d03596c3de\": {\"quality\": 0.5328999999999999, \"cost\": 3.23e-06, \"time\": 0.0322}, \"d07b766487\": {\"quality\": 0.57275, \"cost\": 1.498e-05, \"time\": 0.0336}, \"d0a0a66d75\": {\"quality\": 0.6129000000000001, \"cost\": 1.546e-05, \"time\": 0.033}, \"d0ce31134c\": {\"quality\": 0.5594250000000001, \"cost\": 1.582e-05, \"time\": 0.0356}, \"d0f9633442\": {\"quality\": 0.4102333333333333, \"cost\": 1.92e-06, \"time\": 0.016399999999999998}, \"d216eab7d8\": {\"quality\": 0.5869666666666667, \"cost\": 3.88e-06, \"time\": 0.033699999999999994}, \"d266c19ac8\": {\"quality\": 0.525825, \"cost\": 2.22e-06, \"time\": 0.0281}, \"d26a70179a\": {\"quality\": 0.608975, \"cost\": 1.537e-05, \"time\": 0.034199999999999994}, \"d2af24b59e\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.0224}, \"d2f2dd5cd4\": {\"quality\": 0.5439333333333334, \"cost\": 1.322e-05, \"time\": 0.0197}, \"d302278f85\": {\"quality\": 0.5548, \"cost\": 1.397e-05, \"time\": 0.0262}, \"d37dcaea30\": {\"quality\": 0.6174000000000001, \"cost\": 1.6210000000000002e-05, \"time\": 0.042800000000000005}, \"d3a2d50bd7\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.0183}, \"d3d4185487\": {\"quality\": 0.6016666666666667, \"cost\": 1.462e-05, \"time\": 0.024399999999999998}, \"d3db4cf84d\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"d402233b53\": {\"quality\": 0.608975, \"cost\": 1.537e-05, \"time\": 0.034199999999999994}, \"d43fafa19e\": {\"quality\": 0.41585, \"cost\": 1.5599999999999999e-06, \"time\": 0.0138}, \"d446a75eb7\": {\"quality\": 0.7176, \"cost\": 2.676e-05, \"time\": 0.0297}, \"d48ead13da\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.022399999999999996}, \"d5016f4538\": {\"quality\": 0.41585, \"cost\": 1.5599999999999999e-06, \"time\": 0.0138}, \"d55a3613b0\": {\"quality\": 0.695925, \"cost\": 2.751e-05, \"time\": 0.03950000000000001}, \"d58036ba66\": {\"quality\": 0.5002333333333334, \"cost\": 2.48e-06, \"time\": 0.0224}, \"d5a84c782e\": {\"quality\": 0.6695333333333333, \"cost\": 1.4e-05, \"time\": 0.0275}, \"d5b2eef11c\": {\"quality\": 0.68885, \"cost\": 1.325e-05, \"time\": 0.0177}, \"d6040140b9\": {\"quality\": 0.5955, \"cost\": 5.47e-06, \"time\": 0.048799999999999996}, \"d65185c1a4\": {\"quality\": 0.5680999999999999, \"cost\": 1.86e-06, \"time\": 0.0255}, \"d667351f33\": {\"quality\": 0.6174000000000001, \"cost\": 1.621e-05, \"time\": 0.042800000000000005}, \"d6bd3b66ba\": {\"quality\": 0.4135, \"cost\": 1.08e-06, \"time\": 0.0111}, \"d6c4e48eeb\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.0263}, \"d6cbf265ee\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"d705447fd7\": {\"quality\": 0.608975, \"cost\": 1.537e-05, \"time\": 0.0342}, \"d73a9aab4e\": {\"quality\": 0.41585, \"cost\": 1.5599999999999999e-06, \"time\": 0.0138}, \"d752c30d07\": {\"quality\": 0.5002333333333333, \"cost\": 2.48e-06, \"time\": 0.0224}, \"d782682359\": {\"quality\": 0.5618749999999999, \"cost\": 1.498e-05, \"time\": 0.0303}, \"d7c0972014\": {\"quality\": 0.5082, \"cost\": 4.16e-06, \"time\": 0.0363}, \"d867525748\": {\"quality\": 0.5406666666666666, \"cost\": 1.4060000000000001e-05, \"time\": 0.025}, \"d87eb775da\": {\"quality\": 0.6403333333333333, \"cost\": 3.26e-06, \"time\": 0.0335}, \"d8bab6c09b\": {\"quality\": 0.53045, \"cost\": 4.07e-06, \"time\": 0.0375}, \"d8bcac36e8\": {\"quality\": 0.4969666666666667, \"cost\": 3.32e-06, \"time\": 0.027699999999999995}, \"d8eadc0190\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"d96677d8d4\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.026899999999999997}, \"d98f22270e\": {\"quality\": 0.6129000000000001, \"cost\": 1.546e-05, \"time\": 0.033}, \"d9e2bb21a3\": {\"quality\": 0.5053, \"cost\": 1.47e-06, \"time\": 0.021599999999999998}, \"da95deeb20\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.0289}, \"daaadadcc9\": {\"quality\": 0.4858, \"cost\": 2.84e-06, \"time\": 0.0283}, \"daf855e065\": {\"quality\": 0.525825, \"cost\": 2.22e-06, \"time\": 0.0281}, \"db00594832\": {\"quality\": 0.6129, \"cost\": 1.546e-05, \"time\": 0.033}, \"db19e677c4\": {\"quality\": 0.5149333333333334, \"cost\": 1.322e-05, \"time\": 0.0131}, \"db3c035639\": {\"quality\": 0.626925, \"cost\": 1.6380000000000002e-05, \"time\": 0.0416}, \"db41487005\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.016399999999999998}, \"db6a7482fd\": {\"quality\": 0.68885, \"cost\": 1.325e-05, \"time\": 0.0177}, \"db9060cd27\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"dbce95a072\": {\"quality\": 0.5114666666666666, \"cost\": 3.32e-06, \"time\": 0.031}, \"dc195abe5e\": {\"quality\": 0.5548, \"cost\": 1.397e-05, \"time\": 0.0262}, \"dc3f4b7138\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.016399999999999998}, \"dc66bccb1c\": {\"quality\": 0.5002333333333333, \"cost\": 2.48e-06, \"time\": 0.0224}, \"dc90065dea\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"dd0d70fedd\": {\"quality\": 0.6309, \"cost\": 1.5e-06, \"time\": 0.0196}, \"dddc76b3ca\": {\"quality\": 0.5374, \"cost\": 1.4900000000000001e-05, \"time\": 0.0303}, \"de18bf45e1\": {\"quality\": 0.4425, \"cost\": 3.6e-07, \"time\": 0.0059}, \"de1e56370f\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"df2160ecc8\": {\"quality\": 0.5374, \"cost\": 1.4900000000000001e-05, \"time\": 0.0303}, \"dfda94bd2a\": {\"quality\": 0.42074999999999996, \"cost\": 7.2e-07, \"time\": 0.0085}, \"dff452a9ca\": {\"quality\": 0.4102333333333333, \"cost\": 1.92e-06, \"time\": 0.016399999999999998}, \"e09f75d9d6\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"e0b6a99753\": {\"quality\": 0.7468, \"cost\": 1.25e-05, \"time\": 0.0079}, \"e0cf6587a7\": {\"quality\": 0.50525, \"cost\": 1.4420000000000001e-05, \"time\": 0.0276}, \"e0de4a5929\": {\"quality\": 0.6174000000000001, \"cost\": 1.6210000000000002e-05, \"time\": 0.042800000000000005}, \"e1356fb426\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.0289}, \"e20ba014a1\": {\"quality\": 0.59795, \"cost\": 4.63e-06, \"time\": 0.0435}, \"e21806e3bc\": {\"quality\": 0.5406666666666666, \"cost\": 1.4060000000000001e-05, \"time\": 0.025}, \"e24e97564c\": {\"quality\": 0.66695, \"cost\": 1.576e-05, \"time\": 0.041400000000000006}, \"e2673c1ec8\": {\"quality\": 0.6174, \"cost\": 1.6210000000000002e-05, \"time\": 0.042800000000000005}, \"e26c7bfbdb\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.0197}, \"e2f9980b06\": {\"quality\": 0.6174000000000001, \"cost\": 1.6210000000000002e-05, \"time\": 0.042800000000000005}, \"e3445f7632\": {\"quality\": 0.68885, \"cost\": 1.325e-05, \"time\": 0.0177}, \"e35e5f81a7\": {\"quality\": 0.4763, \"cost\": 1.47e-06, \"time\": 0.015}, \"e376ac53e7\": {\"quality\": 0.5294333333333333, \"cost\": 1.322e-05, \"time\": 0.016399999999999998}, \"e3d8bb56da\": {\"quality\": 0.6067333333333332, \"cost\": 1.361e-05, \"time\": 0.0236}, \"e3df4cf041\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"e47dc3abca\": {\"quality\": 0.5082000000000001, \"cost\": 4.16e-06, \"time\": 0.0363}, \"e4b9d4fb41\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"e510bda989\": {\"quality\": 0.6592, \"cost\": 1.76e-06, \"time\": 0.0139}, \"e517cd2222\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"e51b01f418\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"e520dfae5b\": {\"quality\": 0.47872499999999996, \"cost\": 1.8299999999999998e-06, \"time\": 0.0242}, \"e521c9b7e4\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.0217}, \"e54097ad5d\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"e56a16ca66\": {\"quality\": 0.5367, \"cost\": 1.11e-06, \"time\": 0.0157}, \"e5c4abf7ce\": {\"quality\": 0.5632250000000001, \"cost\": 1.481e-05, \"time\": 0.0348}, \"e5d4689312\": {\"quality\": 0.494375, \"cost\": 1.4420000000000001e-05, \"time\": 0.024300000000000002}, \"e62a7b27ae\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.021699999999999997}, \"e6a7aff3bc\": {\"quality\": 0.543925, \"cost\": 1.397e-05, \"time\": 0.022899999999999997}, \"e736999157\": {\"quality\": 0.543925, \"cost\": 1.397e-05, \"time\": 0.022899999999999997}, \"e7517a8ce0\": {\"quality\": 0.7081666666666666, \"cost\": 2.5750000000000002e-05, \"time\": 0.0256}, \"e7520ca5ac\": {\"quality\": 0.5632250000000001, \"cost\": 1.481e-05, \"time\": 0.0348}, \"e7e94ab7a5\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"e887ddf5cc\": {\"quality\": 0.648825, \"cost\": 2.712e-05, \"time\": 0.0356}, \"e94fb5a295\": {\"quality\": 0.5149333333333334, \"cost\": 1.322e-05, \"time\": 0.0131}, \"ea6ecc5653\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.0236}, \"ea8bcb3ae2\": {\"quality\": 0.5837, \"cost\": 4.7200000000000005e-06, \"time\": 0.03899999999999999}, \"ebbe8b6c4f\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.031}, \"ebdf3abff2\": {\"quality\": 0.6497666666666667, \"cost\": 4.27e-06, \"time\": 0.037599999999999995}, \"ec55dba809\": {\"quality\": 0.561875, \"cost\": 1.498e-05, \"time\": 0.030299999999999997}, \"ecb5f78f37\": {\"quality\": 0.522025, \"cost\": 3.23e-06, \"time\": 0.028899999999999995}, \"ecda3d74cb\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.021699999999999997}, \"ece10c0388\": {\"quality\": 0.6453666666666668, \"cost\": 2.536e-05, \"time\": 0.0217}, \"ed6b5480a5\": {\"quality\": 0.4376, \"cost\": 1.5599999999999999e-06, \"time\": 0.0171}, \"eda630dc85\": {\"quality\": 0.6403333333333333, \"cost\": 3.26e-06, \"time\": 0.0335}, \"edaaee5ed4\": {\"quality\": 0.54595, \"cost\": 2.96e-06, \"time\": 0.025099999999999997}, \"edb2b764aa\": {\"quality\": 0.5869666666666666, \"cost\": 3.88e-06, \"time\": 0.0337}, \"edc52339db\": {\"quality\": 0.47247500000000003, \"cost\": 3.68e-06, \"time\": 0.0303}, \"ede7071775\": {\"quality\": 0.7081666666666667, \"cost\": 2.5750000000000002e-05, \"time\": 0.0256}, \"ee46042c5d\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.0336}, \"ee68c51f73\": {\"quality\": 0.7468, \"cost\": 2.5e-05, \"time\": 0.0158}, \"ee7b726747\": {\"quality\": 0.61985, \"cost\": 1.537e-05, \"time\": 0.0375}, \"eec5f32da9\": {\"quality\": 0.6393, \"cost\": 2.695e-05, \"time\": 0.0368}, \"eed40e4378\": {\"quality\": 0.61985, \"cost\": 1.537e-05, \"time\": 0.0375}, \"eef12d478b\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"ef37b3e0be\": {\"quality\": 0.4183, \"cost\": 2.2799999999999998e-06, \"time\": 0.0223}, \"ef43d497f1\": {\"quality\": 0.48335, \"cost\": 3.68e-06, \"time\": 0.033600000000000005}, \"ef4d4c4a62\": {\"quality\": 0.5869666666666667, \"cost\": 3.88e-06, \"time\": 0.033699999999999994}, \"ef9a651425\": {\"quality\": 0.6740250000000001, \"cost\": 1.677e-05, \"time\": 0.0455}, \"f0655621af\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"f076b4c9ae\": {\"quality\": 0.5630333333333334, \"cost\": 2.87e-06, \"time\": 0.026299999999999997}, \"f11eddb4ed\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"f1408da253\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.0295}, \"f18cf41929\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}, \"f1aa0b0b42\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"f1bda127f6\": {\"quality\": 0.5114666666666666, \"cost\": 3.32e-06, \"time\": 0.031}, \"f1f373e58e\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"f2a2e91541\": {\"quality\": 0.476275, \"cost\": 2.67e-06, \"time\": 0.029500000000000002}, \"f2c04ed1c8\": {\"quality\": 0.4376, \"cost\": 1.5599999999999999e-06, \"time\": 0.0171}, \"f2cf5db12d\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.031}, \"f366c0dd10\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"f4303a5b4f\": {\"quality\": 0.5922000000000001, \"cost\": 2.6560000000000003e-05, \"time\": 0.0329}, \"f437481e3b\": {\"quality\": 0.5399750000000001, \"cost\": 4.24e-06, \"time\": 0.0363}, \"f4bc6b63a7\": {\"quality\": 0.7081666666666666, \"cost\": 2.5750000000000002e-05, \"time\": 0.0256}, \"f4dc556633\": {\"quality\": 0.608975, \"cost\": 1.537e-05, \"time\": 0.0342}, \"f4deb72db6\": {\"quality\": 0.4102333333333333, \"cost\": 1.92e-06, \"time\": 0.016399999999999998}, \"f4ef2b9c33\": {\"quality\": 0.6034666666666667, \"cost\": 1.445e-05, \"time\": 0.0289}, \"f566d6d6a1\": {\"quality\": 0.6161666666666666, \"cost\": 1.462e-05, \"time\": 0.0277}, \"f5b9a94dcc\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"f5c27e7172\": {\"quality\": 0.49079999999999996, \"cost\": 1.47e-06, \"time\": 0.018299999999999997}, \"f5e53d963b\": {\"quality\": 0.4376, \"cost\": 1.5599999999999999e-06, \"time\": 0.0171}, \"f614235c15\": {\"quality\": 0.39899999999999997, \"cost\": 3.6e-07, \"time\": 0.0026}, \"f6546149e3\": {\"quality\": 0.7081666666666666, \"cost\": 2.5750000000000002e-05, \"time\": 0.0256}, \"f74ec023e4\": {\"quality\": 0.494225, \"cost\": 3.68e-06, \"time\": 0.0369}, \"f7b048bd54\": {\"quality\": 0.5742666666666667, \"cost\": 3.71e-06, \"time\": 0.0349}, \"f7c4df993e\": {\"quality\": 0.4763, \"cost\": 1.47e-06, \"time\": 0.015}, \"f854533145\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"f89b8a1930\": {\"quality\": 0.5548, \"cost\": 1.397e-05, \"time\": 0.0262}, \"f93d9a2693\": {\"quality\": 0.5020333333333333, \"cost\": 2.31e-06, \"time\": 0.0269}, \"f97d91a249\": {\"quality\": 0.5406666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.025}, \"f99096d89c\": {\"quality\": 0.543775, \"cost\": 3.23e-06, \"time\": 0.0355}, \"f9e8e221f3\": {\"quality\": 0.43596666666666667, \"cost\": 2.76e-06, \"time\": 0.0283}, \"fa38879eab\": {\"quality\": 0.538875, \"cost\": 4.9100000000000004e-06, \"time\": 0.0461}, \"fa71111570\": {\"quality\": 0.55235, \"cost\": 1.481e-05, \"time\": 0.0315}, \"fa7882d46b\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"fa906520d1\": {\"quality\": 0.5413250000000001, \"cost\": 4.07e-06, \"time\": 0.0408}, \"faabebaa30\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"fb0339a7d0\": {\"quality\": 0.5304500000000001, \"cost\": 4.07e-06, \"time\": 0.0375}, \"fb216ad6b3\": {\"quality\": 0.43270000000000003, \"cost\": 1.2e-06, \"time\": 0.0112}, \"fb6216880a\": {\"quality\": 0.522025, \"cost\": 3.23e-06, \"time\": 0.028899999999999995}, \"fba499b89d\": {\"quality\": 0.561875, \"cost\": 1.498e-05, \"time\": 0.0303}, \"fbc010a368\": {\"quality\": 0.5261666666666667, \"cost\": 1.4060000000000001e-05, \"time\": 0.0217}, \"fbc02e2e07\": {\"quality\": 0.5406666666666666, \"cost\": 1.4060000000000001e-05, \"time\": 0.024999999999999998}, \"fbd6c45271\": {\"quality\": 0.5329, \"cost\": 3.23e-06, \"time\": 0.0322}, \"fc0a156e16\": {\"quality\": 0.6789666666666667, \"cost\": 1.501e-05, \"time\": 0.0316}, \"fc1fd5bf54\": {\"quality\": 0.47872499999999996, \"cost\": 1.8299999999999998e-06, \"time\": 0.0242}, \"fc6967a75b\": {\"quality\": 0.49682499999999996, \"cost\": 1.358e-05, \"time\": 0.019000000000000003}, \"fc73c3b0fa\": {\"quality\": 0.7468, \"cost\": 1.25e-05, \"time\": 0.0079}, \"fce38334b2\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.033600000000000005}, \"fce5fca128\": {\"quality\": 0.6174000000000001, \"cost\": 1.6210000000000002e-05, \"time\": 0.042800000000000005}, \"fd0709359e\": {\"quality\": 0.6309, \"cost\": 7.5e-07, \"time\": 0.0098}, \"fd1f809d64\": {\"quality\": 0.5922333333333333, \"cost\": 1.361e-05, \"time\": 0.0203}, \"fd2c994a9d\": {\"quality\": 0.5548, \"cost\": 1.397e-05, \"time\": 0.026199999999999998}, \"fddccfbf94\": {\"quality\": 0.5114666666666667, \"cost\": 3.32e-06, \"time\": 0.030999999999999996}, \"fe7fa741b4\": {\"quality\": 0.6129000000000001, \"cost\": 1.546e-05, \"time\": 0.033}, \"fe9e1fec71\": {\"quality\": 0.6129, \"cost\": 1.546e-05, \"time\": 0.033}, \"fea4734c09\": {\"quality\": 0.54595, \"cost\": 2.96e-06, \"time\": 0.025099999999999997}, \"fef1ca27fa\": {\"quality\": 0.4917750000000001, \"cost\": 4.52e-06, \"time\": 0.0422}, \"ff11cb6a7a\": {\"quality\": 0.48335000000000006, \"cost\": 3.68e-06, \"time\": 0.0336}, \"ff171e34e2\": {\"quality\": 0.4875333333333334, \"cost\": 2.31e-06, \"time\": 0.023599999999999996}, \"ff1c958e21\": {\"quality\": 0.51495, \"cost\": 1.11e-06, \"time\": 0.0124}, \"ff8df4ace9\": {\"quality\": 0.5374, \"cost\": 1.4900000000000001e-05, \"time\": 0.0303}, \"ff8e68049a\": {\"quality\": 0.42473333333333335, \"cost\": 1.92e-06, \"time\": 0.019700000000000002}}" }, { "path": "abacus-research/cuad-demo.py", "content": "import argparse\nimport json\nimport os\nimport string\n\nimport numpy as np\nimport pandas as pd\nfrom cuad_data_loader import load_cuad_data\n\nimport palimpzest as pz\nfrom palimpzest.constants import Model\n\nCUAD_CATEGORIES = [\n {\n \"Category\": \"Document Name\",\n \"Description\": \"The name of the contract\",\n \"Answer Format\": \"Contract Name\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Parties\",\n \"Description\": \"The two or more parties who signed the contract\",\n \"Answer Format\": \"Entity or individual names\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Agreement Date\",\n \"Description\": \"The date of the contract\",\n \"Answer Format\": \"Date (mm/dd/yyyy)\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Effective Date\",\n \"Description\": \"The date when the contract is effective\\u00a0\",\n \"Answer Format\": \"Date (mm/dd/yyyy)\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Expiration Date\",\n \"Description\": \"On what date will the contract's initial term expire?\",\n \"Answer Format\": \"Date (mm/dd/yyyy) / Perpetual\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Renewal Term\",\n \"Description\": \"What is the renewal term after the initial term expires? This includes automatic extensions and unilateral extensions with prior notice.\",\n \"Answer Format\": \"[Successive] number of years/months / Perpetual\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Notice Period to Terminate Renewal\",\n \"Description\": \"What is the notice period required to terminate renewal?\",\n \"Answer Format\": \"Number of days/months/year(s)\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Governing Law\",\n \"Description\": \"Which state/country's law governs the interpretation of the contract?\",\n \"Answer Format\": \"Name of a US State / non-US Province, Country\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Most Favored Nation\",\n \"Description\": \"Is there a clause that if a third party gets better terms on the licensing or sale of technology/goods/services described in the contract, the buyer of such technology/goods/services under the contract shall be entitled to those better terms?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Non-Compete\",\n \"Description\": \"Is there a restriction on the ability of a party to compete with the counterparty or operate in a certain geography or business or technology sector?\\u00a0\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 2\",\n },\n {\n \"Category\": \"Exclusivity\",\n \"Description\": \"Is there an exclusive dealing\\u00a0 commitment with the counterparty? This includes a commitment to procure all \\u201crequirements\\u201d from one party of certain technology, goods, or services or a prohibition on licensing or selling technology, goods or services to third parties, or a prohibition on\\u00a0 collaborating or working with other parties), whether during the contract or\\u00a0 after the contract ends (or both).\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 2\",\n },\n {\n \"Category\": \"No-Solicit of Customers\",\n \"Description\": \"Is a party restricted from contracting or soliciting customers or partners of the counterparty, whether during the contract or after the contract ends (or both)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 2\",\n },\n {\n \"Category\": \"Competitive Restriction Exception\",\n \"Description\": \"This category includes the exceptions or carveouts to Non-Compete, Exclusivity and No-Solicit of Customers above.\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 2\",\n },\n {\n \"Category\": \"No-Solicit of Employees\",\n \"Description\": \"Is there a restriction on a party\\u2019s soliciting or hiring employees and/or contractors from the\\u00a0 counterparty, whether during the contract or after the contract ends (or both)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Non-Disparagement\",\n \"Description\": \"Is there a requirement on a party not to disparage the counterparty?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Termination for Convenience\",\n \"Description\": \"Can a party terminate this\\u00a0 contract without cause (solely by giving a notice and allowing a waiting\\u00a0 period to expire)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Rofr/Rofo/Rofn\",\n \"Description\": \"Is there a clause granting one party a right of first refusal, right of first offer or right of first negotiation to purchase, license, market, or distribute equity interest, technology, assets, products or services?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Change of Control\",\n \"Description\": \"Does one party have the right to terminate or is consent or notice required of the counterparty if such party undergoes a change of control, such as a merger, stock sale, transfer of all or substantially all of its assets or business, or assignment by operation of law?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 3\",\n },\n {\n \"Category\": \"Anti-Assignment\",\n \"Description\": \"Is consent or notice required of a party if the contract is assigned to a third party?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 3\",\n },\n {\n \"Category\": \"Revenue/Profit Sharing\",\n \"Description\": \"Is one party required to share revenue or profit with the counterparty for any technology, goods, or\\u00a0services?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Price Restrictions\",\n \"Description\": \"Is there a restriction on the\\u00a0 ability of a party to raise or reduce prices of technology, goods, or\\u00a0 services provided?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Minimum Commitment\",\n \"Description\": \"Is there a minimum order size or minimum amount or units per-time period that one party must buy from the counterparty under the contract?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Volume Restriction\",\n \"Description\": \"Is there a fee increase or consent requirement, etc. if one party\\u2019s use of the product/services exceeds certain threshold?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"IP Ownership Assignment\",\n \"Description\": \"Does intellectual property created\\u00a0 by one party become the property of the counterparty, either per the terms of the contract or upon the occurrence of certain events?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Joint IP Ownership\",\n \"Description\": \"Is there any clause providing for joint or shared ownership of intellectual property between the parties to the contract?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"License Grant\",\n \"Description\": \"Does the contract contain a license granted by one party to its counterparty?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Non-Transferable License\",\n \"Description\": \"Does the contract limit the ability of a party to transfer the license being granted to a third party?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Affiliate License-Licensor\",\n \"Description\": \"Does the contract contain a license grant by affiliates of the licensor or that includes intellectual property of affiliates of the licensor?\\u00a0\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Affiliate License-Licensee\",\n \"Description\": \"Does the contract contain a license grant to a licensee (incl. sublicensor) and the affiliates of such licensee/sublicensor?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Unlimited/All-You-Can-Eat-License\",\n \"Description\": \"Is there a clause granting one party an \\u201centerprise,\\u201d \\u201call you can eat\\u201d or unlimited usage license?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Irrevocable or Perpetual License\",\n \"Description\": \"Does the contract contain a\\u00a0 license grant that is irrevocable or perpetual?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Source Code Escrow\",\n \"Description\": \"Is one party required to deposit its source code into escrow with a third party, which can be released to the counterparty upon the occurrence of certain events (bankruptcy,\\u00a0 insolvency, etc.)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Post-Termination Services\",\n \"Description\": \"Is a party subject to obligations after the termination or expiration of a contract, including any post-termination transition, payment, transfer of IP, wind-down, last-buy, or similar commitments?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 5\",\n },\n {\n \"Category\": \"Audit Rights\",\n \"Description\": \"Does a party have the right to\\u00a0 audit the books, records, or physical locations of the counterparty to ensure compliance with the contract?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 5\",\n },\n {\n \"Category\": \"Uncapped Liability\",\n \"Description\": \"Is a party\\u2019s liability uncapped upon the breach of its obligation in the contract? This also includes uncap liability for a particular type of breach such as IP infringement or breach of confidentiality obligation.\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 6\",\n },\n {\n \"Category\": \"Cap on Liability\",\n \"Description\": \"Does the contract include a cap on liability upon the breach of a party\\u2019s obligation? This includes time limitation for the counterparty to bring claims or maximum amount for recovery.\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 6\",\n },\n {\n \"Category\": \"Liquidated Damages\",\n \"Description\": \"Does the contract contain a clause that would award either party liquidated damages for breach or a fee upon the termination of a contract (termination fee)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Warranty Duration\",\n \"Description\": \"What is the duration of any\\u00a0 warranty against defects or errors in technology, products, or services\\u00a0 provided under the contract?\",\n \"Answer Format\": \"Number of months or years\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Insurance\",\n \"Description\": \"Is there a requirement for insurance that must be maintained by one party for the benefit of the counterparty?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Covenant Not to Sue\",\n \"Description\": \"Is a party restricted from contesting the validity of the counterparty\\u2019s ownership of intellectual property or otherwise bringing a claim against the counterparty for matters unrelated to the contract?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Third Party Beneficiary\",\n \"Description\": \"Is there a non-contracting party who is a beneficiary to some or all of the clauses in the contract and therefore can enforce its rights against a contracting party?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n]\n\nNUM_FIELDS_TO_EXTRACT_PER_CONTRACT = 41\n\n# 0.15 is used in the Doc-ETL paper. It should be 0.5 for the actual benchmark.\nIOU_THRESH = 0.15\n\ndef get_label_df(num_contracts: int = 1, seed: int=42) -> pd.DataFrame:\n dataset = load_cuad_data(split=\"test\")\n\n # get the set of unique contract titles; to ensure the order of the contracts is\n # preserved, we use a list rather than using python's set()\n contract_titles = []\n for row in dataset:\n if row[\"title\"] not in contract_titles:\n contract_titles.append(row[\"title\"])\n\n # shuffle the contracts for the given seed\n rng = np.random.default_rng(seed=seed)\n rng.shuffle(contract_titles)\n\n # get the first num_contracts\n contract_titles = contract_titles[:num_contracts]\n\n # construct the dataset one contract at a time\n final_label_dataset = []\n for title in contract_titles:\n # get the rows for this contract\n contract_rows = [row for row in dataset if row[\"title\"] == title]\n\n # construct the contract; we get the contract_id and contract text from the first row\n contract = {\n \"contract_id\": contract_rows[0][\"id\"],\n \"title\": title,\n \"contract\": contract_rows[0][\"context\"],\n }\n\n # add the labels\n category_names = list(map(lambda category: category[\"Category\"], CUAD_CATEGORIES))\n contract.update({category_name: [] for category_name in category_names})\n for row in contract_rows:\n category_name = row[\"id\"].split(\"__\")[-1].split(\"_\")[0].strip()\n category_name = category_name.replace(\" For \", \" for \")\n category_name = category_name.replace(\" Of \", \" of \")\n category_name = category_name.replace(\" On \", \" on \")\n category_name = category_name.replace(\" Or \", \" or \")\n category_name = category_name.replace(\" To \", \" to \")\n category_name = category_name.replace(\"Ip\", \"IP\")\n assert category_name in category_names, f\"Unknown category {category_name}\"\n\n # Extract text from answers list (handles both old and new format)\n answer_texts = []\n if isinstance(row[\"answers\"], list):\n answer_texts = [ans[\"text\"] for ans in row[\"answers\"]] if row[\"answers\"] else []\n else:\n answer_texts = row[\"answers\"].get(\"text\", [])\n contract[category_name].extend(answer_texts)\n\n # add the contract to the dataset\n final_label_dataset.append(contract)\n\n return pd.DataFrame(final_label_dataset)\n\n\n# Return the Jaccard similarity between two strings\ndef get_jaccard(label, pred):\n remove_tokens = [c for c in string.punctuation if c != \"/\"]\n for token in remove_tokens:\n label = label.replace(token, \"\")\n pred = pred.replace(token, \"\")\n label = label.lower()\n pred = pred.lower()\n label = label.replace(\"/\", \" \")\n pred = pred.replace(\"/\", \" \")\n\n label_words = set(label.split(\" \"))\n pred_words = set(pred.split(\" \"))\n\n intersection = label_words.intersection(pred_words)\n union = label_words.union(pred_words)\n jaccard = len(intersection) / len(union)\n return jaccard\n\n\n# Find the number of true positives, false positives, and false negatives for each entry\n# (one field extracted from each contract) by comparing the labels and predictions.\n# Labels and preds are lists of strings\ndef evaluate_entry(labels, preds, substr_ok):\n tp, fp, fn = 0, 0, 0\n\n # jaccard similarity expects strings\n # TODO: This is a hack, ideally, the return type of the preds should be known\n for idx, pred in enumerate(preds):\n if not isinstance(pred, str):\n print(f\"Expected string, but got {pred}\")\n preds[idx] = str(pred)\n\n # first check if labels is empty\n if len(labels) == 0:\n if len(preds) > 0:\n fp += len(preds) # false positive for each one\n else:\n for ans in labels:\n assert len(ans) > 0\n # check if there is a match\n match_found = False\n for pred in preds:\n if substr_ok:\n is_match = get_jaccard(ans, pred) >= IOU_THRESH or ans in pred\n else:\n is_match = get_jaccard(ans, pred) >= IOU_THRESH\n if is_match:\n match_found = True\n\n if match_found:\n tp += 1\n else:\n fn += 1\n\n # now also get any fps by looping through preds\n for pred in preds:\n # Check if there's a match. if so, don't count (don't want to double count based on the above)\n # but if there's no match, then this is a false positive.\n # (Note: we get the true positives in the above loop instead of this loop so that we don't double count\n # multiple predictions that are matched with the same answer.)\n match_found = False\n for ans in labels:\n assert len(ans) > 0\n if substr_ok:\n is_match = get_jaccard(ans, pred) >= IOU_THRESH or ans in pred\n else:\n is_match = get_jaccard(ans, pred) >= IOU_THRESH\n if is_match:\n match_found = True\n\n if not match_found:\n fp += 1\n\n return tp, fp, fn\n\n\ndef handle_empty_preds(preds):\n if preds is None or ( # noqa: SIM114\n isinstance(preds, str) and (preds == \"\" or preds == \" \" or preds == \"null\" or preds == \"None\")\n ):\n return []\n elif isinstance(preds, float) and np.isnan(preds):\n return []\n if not isinstance(preds, (list, np.ndarray)):\n return [preds]\n return preds\n\n\nclass CUADValidator(pz.Validator):\n def __init__(self, num_contracts: int = 1, seed: int=42):\n super().__init__()\n self.num_contracts = num_contracts\n self.seed = seed\n\n # get clean names for the categories\n self.category_names = list(map(lambda category: category[\"Category\"], CUAD_CATEGORIES))\n\n # compute mapping from contract_id --> label\n self.contract_id_to_label = self._compute_contract_id_to_labels()\n\n def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n tps, fps, fns = 0, 0, 0\n for field in fields:\n preds = handle_empty_preds(output.get(field))\n labels = self.contract_id_to_label[input_record[\"contract_id\"]][field]\n entry_tp, entry_fp, entry_fn = evaluate_entry(labels, preds, substr_ok=True) if field == \"Parties\" else evaluate_entry(labels, preds, substr_ok=False)\n tps += entry_tp\n fps += entry_fp\n fns += entry_fn\n precision = tps / (tps + fps) if tps + fps > 0 else 0.0\n recall = tps / (tps + fns) if tps + fns > 0 else 0.0\n f1 = 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0.0\n\n return f1\n\n def _compute_contract_id_to_labels(self):\n # load full train dataset\n dataset = load_cuad_data(split=\"train\")\n\n # get the set of unique contract titles; to ensure the order of the contracts is\n # preserved, we use a list rather than using python's set()\n contract_titles = []\n for row in dataset:\n if row[\"title\"] not in contract_titles:\n contract_titles.append(row[\"title\"])\n\n # shuffle the contracts for the given seed\n rng = np.random.default_rng(seed=self.seed)\n rng.shuffle(contract_titles)\n\n # get the first num_contracts\n contract_titles = contract_titles[:self.num_contracts]\n\n # construct the mapping from contract_id to labels\n contract_id_to_labels = {}\n for title in contract_titles:\n # get the rows for this contract\n contract_rows = [row for row in dataset if row[\"title\"] == title]\n\n # get the contract_id from the first row\n contract_id = contract_rows[0][\"id\"]\n\n # get the labels\n labels = {category: [] for category in self.category_names}\n for row in contract_rows:\n category_name = row[\"id\"].split(\"__\")[-1].split(\"_\")[0].strip()\n category_name = category_name.replace(\" For \", \" for \")\n category_name = category_name.replace(\" Of \", \" of \")\n category_name = category_name.replace(\" On \", \" on \")\n category_name = category_name.replace(\" Or \", \" or \")\n category_name = category_name.replace(\" To \", \" to \")\n category_name = category_name.replace(\"Ip\", \"IP\")\n assert category_name in self.category_names, f\"Unknown category {category_name}\"\n\n # Extract text from answers list (handles both old and new format)\n answer_texts = []\n if isinstance(row[\"answers\"], list):\n answer_texts = [ans[\"text\"] for ans in row[\"answers\"]] if row[\"answers\"] else []\n else:\n answer_texts = row[\"answers\"].get(\"text\", [])\n labels[category_name].extend(answer_texts)\n\n # update the dictionary\n contract_id_to_labels[contract_id] = labels\n\n return contract_id_to_labels\n\n\nclass CUADDataset(pz.IterDataset):\n def __init__(self, num_contracts: int = 1, split: str = \"train\", seed: int=42):\n self.num_contracts = num_contracts\n self.split = split\n self.seed = seed\n\n input_cols = [\n {\"name\": \"contract_id\", \"type\": str, \"desc\": \"The id of the the contract to be analyzed\"},\n {\"name\": \"title\", \"type\": str, \"desc\": \"The title of the the contract to be analyzed\"},\n {\"name\": \"contract\", \"type\": str, \"desc\": \"The content of the the contract to be analyzed\"},\n ]\n super().__init__(id=\"cuad\", schema=input_cols)\n\n # convert the dataset into a list of dictionaries where each row is for a single contract\n dataset = load_cuad_data(split=split)\n self.dataset = self._construct_dataset(dataset, num_contracts, seed)\n\n\n def _construct_dataset(self, dataset, num_contracts, seed: int=42):\n # get the set of unique contract titles; to ensure the order of the contracts is\n # preserved, we use a list rather than using python's set()\n contract_titles = []\n for row in dataset:\n if row[\"title\"] not in contract_titles:\n contract_titles.append(row[\"title\"])\n\n # shuffle the contracts for the given seed\n rng = np.random.default_rng(seed=seed)\n rng.shuffle(contract_titles)\n\n # get the first num_contracts\n contract_titles = contract_titles[:num_contracts]\n\n # construct the dataset one contract at a time\n new_dataset = []\n for title in contract_titles:\n # get the rows for this contract\n contract_rows = [row for row in dataset if row[\"title\"] == title]\n\n # construct the contract; we get the contract_id and contract text from the first row\n contract = {\n \"contract_id\": contract_rows[0][\"id\"],\n \"title\": title,\n \"contract\": contract_rows[0][\"context\"],\n }\n\n # add the rows to the dataset\n new_dataset.append(contract)\n\n return new_dataset\n\n def __len__(self):\n return self.num_contracts\n\n def __getitem__(self, idx: int):\n return self.dataset[idx]\n\n# Compute the precision and recall for the entire dataset.\n# Each row in the dataframes should correspond to a contract.\n# The columns should be the extracted fields (categories in CUAD_CATEGORIES).\ndef compute_precision_recall(label_df, preds_df):\n tp, fp, fn = 0, 0, 0\n\n label_df = label_df.sort_values(\"contract_id\").reset_index(drop=True)\n preds_df = preds_df.sort_values(\"contract_id\").reset_index(drop=True)\n\n assert label_df.shape == preds_df.shape, (\n f\"Label and prediction dataframes have different shapes, label shape: {label_df.shape} vs preds shape {preds_df.shape}\"\n )\n\n categories = [category[\"Category\"] for category in CUAD_CATEGORIES]\n\n for label_row, pred_row in zip(label_df.iterrows(), preds_df.iterrows()):\n assert label_row[1][\"contract_id\"] == pred_row[1][\"contract_id\"], (\n f\"IDs do not match. label id: {label_row[1]['contract_id']} vs pred id: {pred_row[1]['contract_id']}\"\n )\n for category in categories:\n substr_ok = \"Parties\" in category\n\n labels = label_row[1][category]\n assert isinstance(labels, list)\n\n preds = pred_row[1][category]\n preds = handle_empty_preds(preds)\n\n entry_tp, entry_fp, entry_fn = evaluate_entry(labels, preds, substr_ok)\n tp += entry_tp\n fp += entry_fp\n fn += entry_fn\n\n precision = tp / (tp + fp) if tp + fp > 0 else np.nan\n recall = tp / (tp + fn) if tp + fn > 0 else np.nan\n\n return precision, recall\n\n\ndef parse_arguments():\n parser = argparse.ArgumentParser(description=\"Run CUAD demo\")\n parser.add_argument(\"--mode\", type=str, help=\"one-convert or separate-converts\", default=\"one-convert\")\n parser.add_argument(\"--test\", type=str, help=\"test time compute active or inactive\", default=\"active\")\n parser.add_argument(\"--constrained\", default=False, action=\"store_true\", help=\"Use constrained objective\")\n parser.add_argument(\"--gpt4-mini-only\", default=False, action=\"store_true\", help=\"Use only GPT-4o-mini\")\n parser.add_argument(\n \"--sentinel-execution-strategy\",\n default=\"mab\",\n type=str,\n help=\"The engine to use. One of mab or random\",\n )\n parser.add_argument(\n \"--execution-strategy\",\n default=\"parallel\",\n type=str,\n help=\"The plan executor to use. One of sequential, pipelined, parallel\",\n )\n parser.add_argument(\n \"--optimizer-strategy\",\n default=\"pareto\",\n type=str,\n help=\"The optimizer to use. One of pareto or greedy\",\n )\n parser.add_argument(\"--verbose\", default=False, action=\"store_true\", help=\"Print verbose output\")\n parser.add_argument(\n \"--seed\",\n default=42,\n type=int,\n help=\"Seed used to initialize RNG for MAB sampling algorithm\",\n )\n parser.add_argument(\n \"--k\",\n default=10,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--j\",\n default=3,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--sample-budget\",\n default=100,\n type=int,\n help=\"Total sample budget in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--exp-name\",\n default=None,\n type=str,\n help=\"The experiment name.\",\n )\n parser.add_argument(\n \"--priors-file\",\n default=None,\n type=str,\n help=\"A file with a dictionary mapping physical operator ids to prior belief on their performance\",\n )\n parser.add_argument(\n \"--quality\",\n default=None,\n type=float,\n help=\"Quality threshold\",\n )\n parser.add_argument(\n \"--policy\",\n default=\"maxquality\",\n type=str,\n help=\"One of 'mincost', 'mintime', 'maxquality'\",\n )\n return parser.parse_args()\n\n\ndef build_cuad_query(dataset, mode):\n assert mode in [\"one-convert\", \"separate-converts\"]\n\n if mode == \"one-convert\":\n cols = []\n for category in CUAD_CATEGORIES:\n desc = (\n f\"Extract the text spans (if they exist) from the contract corresponding to: {category['Description']}. If no spans exist, return an empty list. Quote text spans verbatim (do not summarize or paraphrase).\"\n )\n cols.append({\"name\": category[\"Category\"], \"type\": list[str], \"desc\": desc})\n\n desc = \"Extract the text spans (if they exist) from the contract.\"\n dataset = dataset.sem_map(cols, depends_on=[\"contract\"])\n elif mode == \"separate-converts\":\n for category in CUAD_CATEGORIES:\n desc = (\n f\"Extract the text spans (if they exist) from the contract corresponding to: {category['Description']}. If no spans exist, return an empty list. Quote text spans verbatim (do not summarize or paraphrase).\"\n )\n dataset = dataset.sem_map(\n [{\"name\": category[\"Category\"], \"type\": list[str], \"desc\": desc}],\n depends_on=[\"contract\"],\n )\n\n return dataset\n\n\ndef main():\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n args = parse_arguments()\n\n # create directory for profiling data\n os.makedirs(\"opt-profiling-data\", exist_ok=True)\n\n # create validator for CUAD\n validator = CUADValidator(num_contracts=25, seed=args.seed)\n\n # create datasets for CUAD\n dataset = CUADDataset(split=\"test\", num_contracts=100, seed=args.seed)\n train_dataset = CUADDataset(split=\"train\", num_contracts=25, seed=args.seed)\n train_dataset = {train_dataset.id: train_dataset}\n print(\"Created datasets\")\n\n # build and run the CUAD query\n query = build_cuad_query(dataset, args.mode)\n print(\"Built query; Starting query execution\")\n\n # set the optimization policy; constraint set to 25% percentile from unconstrained plans\n policy = pz.MaxQuality() if not args.constrained else pz.MaxQualityAtFixedCost(max_cost=2.759)\n if args.policy == \"mincost\":\n policy = pz.MinCost()\n elif args.policy == \"minlatency\":\n policy = pz.MinTime()\n elif args.quality is not None and args.policy == \"mincostatfixedquality\":\n policy = pz.MinCostAtFixedQuality(min_quality=args.quality)\n elif args.quality is not None and args.policy == \"minlatencyatfixedquality\":\n policy = pz.MinTimeAtFixedQuality(min_quality=args.quality)\n print(f\"USING POLICY: {policy}\")\n\n # set models\n models = [Model.GPT_4o_MINI] if args.gpt4_mini_only else [\n Model.GPT_4o,\n Model.GPT_4o_MINI,\n Model.LLAMA3_1_8B,\n Model.LLAMA3_3_70B,\n # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments`\n Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B,\n ]\n\n sentinel_strategy = args.sentinel_execution_strategy\n optimizer_strategy = args.optimizer_strategy\n execution_strategy = args.execution_strategy\n seed = args.seed\n k = args.k\n j = args.j\n sample_budget = args.sample_budget\n exp_name = (\n f\"cuad-final-{sentinel_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}\"\n if args.exp_name is None\n else args.exp_name\n )\n priors = None\n if args.priors_file is not None:\n with open(args.priors_file) as f:\n priors = json.load(f)\n\n config = pz.QueryProcessorConfig(\n policy=policy,\n verbose=False,\n optimizer_strategy=optimizer_strategy,\n sentinel_execution_strategy=sentinel_strategy,\n execution_strategy=execution_strategy,\n max_workers=64,\n available_models=models,\n allow_bonded_query=True,\n allow_critic=True,\n allow_mixtures=True,\n allow_rag_reduction=True,\n progress=True,\n k=k,\n j=j,\n sample_budget=sample_budget,\n seed=seed,\n exp_name=exp_name,\n priors=priors,\n dont_use_priors=(priors is None),\n )\n\n print(f\"EXPERIMENT NAME: {exp_name}\")\n data_record_collection = query.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator)\n print(\"Query execution completed\")\n\n # save statistics\n execution_stats_dict = data_record_collection.execution_stats.to_json()\n with open(f\"opt-profiling-data/{exp_name}-stats.json\", \"w\") as f:\n json.dump(execution_stats_dict, f)\n\n pred_df = data_record_collection.to_df()\n label_df = get_label_df(num_contracts=100, seed=seed)\n # pred_df.to_csv(f\"{exp_name}-pred.csv\", index=False)\n # label_df.to_csv(f\"{exp_name}-label.csv\", index=False)\n final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0]\n final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id]\n\n prec, recall = compute_precision_recall(label_df, pred_df)\n f1 = 2 * (prec * recall) / (prec + recall) if prec + recall > 0 else 0.0\n stats_dict = {\n \"precision\": prec,\n \"recall\": recall,\n \"f1\": f1,\n \"optimization_time\": data_record_collection.execution_stats.optimization_time,\n \"optimization_cost\": data_record_collection.execution_stats.optimization_cost,\n \"plan_execution_time\": data_record_collection.execution_stats.plan_execution_time,\n \"plan_execution_cost\": data_record_collection.execution_stats.plan_execution_cost,\n \"total_execution_time\": data_record_collection.execution_stats.total_execution_time,\n \"total_execution_cost\": data_record_collection.execution_stats.total_execution_cost,\n \"plan_str\": final_plan_str,\n }\n with open(f\"opt-profiling-data/{exp_name}-metrics.json\", \"w\") as f:\n json.dump(stats_dict, f)\n\n print(f\"Precision: {prec:.3f}, Recall: {recall:.3f}, F1: {f1:.3f}\")\n print(f\"Optimization time: {data_record_collection.execution_stats.optimization_time}\")\n print(f\"Optimization cost: {data_record_collection.execution_stats.optimization_cost}\")\n print(f\"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}\")\n print(f\"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}\")\n print(f\"Total Execution time: {data_record_collection.execution_stats.total_execution_time}\")\n print(f\"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "abacus-research/cuad-max-quality-at-cost.py", "content": "import argparse\nimport json\nimport os\nimport string\n\nimport numpy as np\nimport pandas as pd\nfrom cuad_data_loader import load_cuad_data\n\nimport palimpzest as pz\nfrom palimpzest.constants import Model\nfrom palimpzest.policy import MaxQuality, MaxQualityAtFixedCost\n\nCUAD_CATEGORIES = [\n {\n \"Category\": \"Document Name\",\n \"Description\": \"The name of the contract\",\n \"Answer Format\": \"Contract Name\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Parties\",\n \"Description\": \"The two or more parties who signed the contract\",\n \"Answer Format\": \"Entity or individual names\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Agreement Date\",\n \"Description\": \"The date of the contract\",\n \"Answer Format\": \"Date (mm/dd/yyyy)\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Effective Date\",\n \"Description\": \"The date when the contract is effective\\u00a0\",\n \"Answer Format\": \"Date (mm/dd/yyyy)\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Expiration Date\",\n \"Description\": \"On what date will the contract's initial term expire?\",\n \"Answer Format\": \"Date (mm/dd/yyyy) / Perpetual\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Renewal Term\",\n \"Description\": \"What is the renewal term after the initial term expires? This includes automatic extensions and unilateral extensions with prior notice.\",\n \"Answer Format\": \"[Successive] number of years/months / Perpetual\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Notice Period to Terminate Renewal\",\n \"Description\": \"What is the notice period required to terminate renewal?\",\n \"Answer Format\": \"Number of days/months/year(s)\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Governing Law\",\n \"Description\": \"Which state/country's law governs the interpretation of the contract?\",\n \"Answer Format\": \"Name of a US State / non-US Province, Country\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Most Favored Nation\",\n \"Description\": \"Is there a clause that if a third party gets better terms on the licensing or sale of technology/goods/services described in the contract, the buyer of such technology/goods/services under the contract shall be entitled to those better terms?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Non-Compete\",\n \"Description\": \"Is there a restriction on the ability of a party to compete with the counterparty or operate in a certain geography or business or technology sector?\\u00a0\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 2\",\n },\n {\n \"Category\": \"Exclusivity\",\n \"Description\": \"Is there an exclusive dealing\\u00a0 commitment with the counterparty? This includes a commitment to procure all \\u201crequirements\\u201d from one party of certain technology, goods, or services or a prohibition on licensing or selling technology, goods or services to third parties, or a prohibition on\\u00a0 collaborating or working with other parties), whether during the contract or\\u00a0 after the contract ends (or both).\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 2\",\n },\n {\n \"Category\": \"No-Solicit of Customers\",\n \"Description\": \"Is a party restricted from contracting or soliciting customers or partners of the counterparty, whether during the contract or after the contract ends (or both)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 2\",\n },\n {\n \"Category\": \"Competitive Restriction Exception\",\n \"Description\": \"This category includes the exceptions or carveouts to Non-Compete, Exclusivity and No-Solicit of Customers above.\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 2\",\n },\n {\n \"Category\": \"No-Solicit of Employees\",\n \"Description\": \"Is there a restriction on a party\\u2019s soliciting or hiring employees and/or contractors from the\\u00a0 counterparty, whether during the contract or after the contract ends (or both)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Non-Disparagement\",\n \"Description\": \"Is there a requirement on a party not to disparage the counterparty?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Termination for Convenience\",\n \"Description\": \"Can a party terminate this\\u00a0 contract without cause (solely by giving a notice and allowing a waiting\\u00a0 period to expire)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Rofr/Rofo/Rofn\",\n \"Description\": \"Is there a clause granting one party a right of first refusal, right of first offer or right of first negotiation to purchase, license, market, or distribute equity interest, technology, assets, products or services?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Change of Control\",\n \"Description\": \"Does one party have the right to terminate or is consent or notice required of the counterparty if such party undergoes a change of control, such as a merger, stock sale, transfer of all or substantially all of its assets or business, or assignment by operation of law?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 3\",\n },\n {\n \"Category\": \"Anti-Assignment\",\n \"Description\": \"Is consent or notice required of a party if the contract is assigned to a third party?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 3\",\n },\n {\n \"Category\": \"Revenue/Profit Sharing\",\n \"Description\": \"Is one party required to share revenue or profit with the counterparty for any technology, goods, or\\u00a0services?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Price Restrictions\",\n \"Description\": \"Is there a restriction on the\\u00a0 ability of a party to raise or reduce prices of technology, goods, or\\u00a0 services provided?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Minimum Commitment\",\n \"Description\": \"Is there a minimum order size or minimum amount or units per-time period that one party must buy from the counterparty under the contract?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Volume Restriction\",\n \"Description\": \"Is there a fee increase or consent requirement, etc. if one party\\u2019s use of the product/services exceeds certain threshold?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"IP Ownership Assignment\",\n \"Description\": \"Does intellectual property created\\u00a0 by one party become the property of the counterparty, either per the terms of the contract or upon the occurrence of certain events?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Joint IP Ownership\",\n \"Description\": \"Is there any clause providing for joint or shared ownership of intellectual property between the parties to the contract?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"License Grant\",\n \"Description\": \"Does the contract contain a license granted by one party to its counterparty?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Non-Transferable License\",\n \"Description\": \"Does the contract limit the ability of a party to transfer the license being granted to a third party?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Affiliate License-Licensor\",\n \"Description\": \"Does the contract contain a license grant by affiliates of the licensor or that includes intellectual property of affiliates of the licensor?\\u00a0\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Affiliate License-Licensee\",\n \"Description\": \"Does the contract contain a license grant to a licensee (incl. sublicensor) and the affiliates of such licensee/sublicensor?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Unlimited/All-You-Can-Eat-License\",\n \"Description\": \"Is there a clause granting one party an \\u201centerprise,\\u201d \\u201call you can eat\\u201d or unlimited usage license?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Irrevocable or Perpetual License\",\n \"Description\": \"Does the contract contain a\\u00a0 license grant that is irrevocable or perpetual?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Source Code Escrow\",\n \"Description\": \"Is one party required to deposit its source code into escrow with a third party, which can be released to the counterparty upon the occurrence of certain events (bankruptcy,\\u00a0 insolvency, etc.)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Post-Termination Services\",\n \"Description\": \"Is a party subject to obligations after the termination or expiration of a contract, including any post-termination transition, payment, transfer of IP, wind-down, last-buy, or similar commitments?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 5\",\n },\n {\n \"Category\": \"Audit Rights\",\n \"Description\": \"Does a party have the right to\\u00a0 audit the books, records, or physical locations of the counterparty to ensure compliance with the contract?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 5\",\n },\n {\n \"Category\": \"Uncapped Liability\",\n \"Description\": \"Is a party\\u2019s liability uncapped upon the breach of its obligation in the contract? This also includes uncap liability for a particular type of breach such as IP infringement or breach of confidentiality obligation.\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 6\",\n },\n {\n \"Category\": \"Cap on Liability\",\n \"Description\": \"Does the contract include a cap on liability upon the breach of a party\\u2019s obligation? This includes time limitation for the counterparty to bring claims or maximum amount for recovery.\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 6\",\n },\n {\n \"Category\": \"Liquidated Damages\",\n \"Description\": \"Does the contract contain a clause that would award either party liquidated damages for breach or a fee upon the termination of a contract (termination fee)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Warranty Duration\",\n \"Description\": \"What is the duration of any\\u00a0 warranty against defects or errors in technology, products, or services\\u00a0 provided under the contract?\",\n \"Answer Format\": \"Number of months or years\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Insurance\",\n \"Description\": \"Is there a requirement for insurance that must be maintained by one party for the benefit of the counterparty?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Covenant Not to Sue\",\n \"Description\": \"Is a party restricted from contesting the validity of the counterparty\\u2019s ownership of intellectual property or otherwise bringing a claim against the counterparty for matters unrelated to the contract?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Third Party Beneficiary\",\n \"Description\": \"Is there a non-contracting party who is a beneficiary to some or all of the clauses in the contract and therefore can enforce its rights against a contracting party?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n]\n\nNUM_FIELDS_TO_EXTRACT_PER_CONTRACT = 41\n\n# 0.15 is used in the Doc-ETL paper. It should be 0.5 for the actual benchmark.\nIOU_THRESH = 0.15\n\ndef get_label_df(num_contracts: int = 1, seed: int=42) -> pd.DataFrame:\n dataset = load_cuad_data(split=\"test\")\n\n # get the set of unique contract titles; to ensure the order of the contracts is\n # preserved, we use a list rather than using python's set()\n contract_titles = []\n for row in dataset:\n if row[\"title\"] not in contract_titles:\n contract_titles.append(row[\"title\"])\n\n # shuffle the contracts for the given seed\n rng = np.random.default_rng(seed=seed)\n rng.shuffle(contract_titles)\n\n # get the first num_contracts\n contract_titles = contract_titles[:num_contracts]\n\n # construct the dataset one contract at a time\n final_label_dataset = []\n for title in contract_titles:\n # get the rows for this contract\n contract_rows = [row for row in dataset if row[\"title\"] == title]\n\n # construct the contract; we get the contract_id and contract text from the first row\n contract = {\n \"contract_id\": contract_rows[0][\"id\"],\n \"title\": title,\n \"contract\": contract_rows[0][\"context\"],\n }\n\n # add the labels\n category_names = list(map(lambda category: category[\"Category\"], CUAD_CATEGORIES))\n for row in contract_rows:\n category_name = row[\"id\"].split(\"__\")[-1].split(\"_\")[0].strip()\n category_name = category_name.replace(\" For \", \" for \")\n category_name = category_name.replace(\" Of \", \" of \")\n category_name = category_name.replace(\" On \", \" on \")\n category_name = category_name.replace(\" Or \", \" or \")\n category_name = category_name.replace(\" To \", \" to \")\n category_name = category_name.replace(\"Ip\", \"IP\")\n assert category_name in category_names, f\"Unknown category {category_name}\"\n\n # Extract text from answers list (handles both old and new format)\n answer_texts = []\n if isinstance(row[\"answers\"], list):\n answer_texts = [ans[\"text\"] for ans in row[\"answers\"]] if row[\"answers\"] else []\n else:\n answer_texts = row[\"answers\"].get(\"text\", [])\n contract[category_name].extend(answer_texts)\n\n # add the contract to the dataset\n final_label_dataset.append(contract)\n\n return pd.DataFrame(final_label_dataset)\n\n\n# Return the Jaccard similarity between two strings\ndef get_jaccard(label, pred):\n remove_tokens = [c for c in string.punctuation if c != \"/\"]\n for token in remove_tokens:\n label = label.replace(token, \"\")\n pred = pred.replace(token, \"\")\n label = label.lower()\n pred = pred.lower()\n label = label.replace(\"/\", \" \")\n pred = pred.replace(\"/\", \" \")\n\n label_words = set(label.split(\" \"))\n pred_words = set(pred.split(\" \"))\n\n intersection = label_words.intersection(pred_words)\n union = label_words.union(pred_words)\n jaccard = len(intersection) / len(union)\n return jaccard\n\n\n# Find the number of true positives, false positives, and false negatives for each entry\n# (one field extracted from each contract) by comparing the labels and predictions.\n# Labels and preds are lists of strings\ndef evaluate_entry(labels, preds, substr_ok):\n tp, fp, fn = 0, 0, 0\n\n # jaccard similarity expects strings\n # TODO: This is a hack, ideally, the return type of the preds should be known\n for idx, pred in enumerate(preds):\n if not isinstance(pred, str):\n print(f\"Expected string, but got {pred}\")\n preds[idx] = str(pred)\n\n # first check if labels is empty\n if len(labels) == 0:\n if len(preds) > 0:\n fp += len(preds) # false positive for each one\n else:\n for ans in labels:\n assert len(ans) > 0\n # check if there is a match\n match_found = False\n for pred in preds:\n if substr_ok:\n is_match = get_jaccard(ans, pred) >= IOU_THRESH or ans in pred\n else:\n is_match = get_jaccard(ans, pred) >= IOU_THRESH\n if is_match:\n match_found = True\n\n if match_found:\n tp += 1\n else:\n fn += 1\n\n # now also get any fps by looping through preds\n for pred in preds:\n # Check if there's a match. if so, don't count (don't want to double count based on the above)\n # but if there's no match, then this is a false positive.\n # (Note: we get the true positives in the above loop instead of this loop so that we don't double count\n # multiple predictions that are matched with the same answer.)\n match_found = False\n for ans in labels:\n assert len(ans) > 0\n if substr_ok:\n is_match = get_jaccard(ans, pred) >= IOU_THRESH or ans in pred\n else:\n is_match = get_jaccard(ans, pred) >= IOU_THRESH\n if is_match:\n match_found = True\n\n if not match_found:\n fp += 1\n\n return tp, fp, fn\n\n\ndef handle_empty_preds(preds):\n if preds is None or ( # noqa: SIM114\n isinstance(preds, str) and (preds == \"\" or preds == \" \" or preds == \"null\" or preds == \"None\")\n ):\n return []\n elif isinstance(preds, float) and np.isnan(preds):\n return []\n if not isinstance(preds, (list, np.ndarray)):\n return [preds]\n return preds\n\n\nclass CUADValidator(pz.Validator):\n def __init__(self, num_contracts: int = 1, seed: int=42):\n super().__init__()\n self.num_contracts = num_contracts\n self.seed = seed\n\n # get clean names for the categories\n self.category_names = list(map(lambda category: category[\"Category\"], CUAD_CATEGORIES))\n\n # compute mapping from contract_id --> label\n self.contract_id_to_label = self._compute_contract_id_to_labels()\n\n def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n scores = []\n for field in fields:\n preds = handle_empty_preds(output.get(field))\n labels = self.contract_id_to_label[input_record[\"contract_id\"]][field]\n entry_tp, _, entry_fn = evaluate_entry(labels, preds, substr_ok=True) if field == \"Parties\" else evaluate_entry(labels, preds, substr_ok=False)\n score = None\n if len(labels) > 0: # noqa: SIM108\n score = entry_tp / (entry_tp + entry_fn)\n else:\n score = 1.0 if len(preds) == 0 else 0.0\n scores.append(score)\n return np.mean(scores)\n\n def _compute_contract_id_to_labels(self):\n # load full train dataset\n dataset = load_cuad_data(split=\"train\")\n\n # get the set of unique contract titles; to ensure the order of the contracts is\n # preserved, we use a list rather than using python's set()\n contract_titles = []\n for row in dataset:\n if row[\"title\"] not in contract_titles:\n contract_titles.append(row[\"title\"])\n\n # shuffle the contracts for the given seed\n rng = np.random.default_rng(seed=self.seed)\n rng.shuffle(contract_titles)\n\n # get the first num_contracts\n contract_titles = contract_titles[:self.num_contracts]\n\n # construct the mapping from contract_id to labels\n contract_id_to_labels = {}\n for title in contract_titles:\n # get the rows for this contract\n contract_rows = [row for row in dataset if row[\"title\"] == title]\n\n # get the contract_id from the first row\n contract_id = contract_rows[0][\"id\"]\n\n # get the labels\n labels = {category: [] for category in self.category_names}\n for row in contract_rows:\n category_name = row[\"id\"].split(\"__\")[-1].split(\"_\")[0].strip()\n category_name = category_name.replace(\" For \", \" for \")\n category_name = category_name.replace(\" Of \", \" of \")\n category_name = category_name.replace(\" On \", \" on \")\n category_name = category_name.replace(\" Or \", \" or \")\n category_name = category_name.replace(\" To \", \" to \")\n category_name = category_name.replace(\"Ip\", \"IP\")\n assert category_name in self.category_names, f\"Unknown category {category_name}\"\n\n # Extract text from answers list (handles both old and new format)\n answer_texts = []\n if isinstance(row[\"answers\"], list):\n answer_texts = [ans[\"text\"] for ans in row[\"answers\"]] if row[\"answers\"] else []\n else:\n answer_texts = row[\"answers\"].get(\"text\", [])\n labels[category_name].extend(answer_texts)\n\n # update the dictionary\n contract_id_to_labels[contract_id] = labels\n\n return contract_id_to_labels\n\n\nclass CUADDataset(pz.IterDataset):\n def __init__(self, num_contracts: int = 1, split: str = \"train\", seed: int=42):\n self.num_contracts = num_contracts\n self.split = split\n self.seed = seed\n\n input_cols = [\n {\"name\": \"contract_id\", \"type\": str, \"desc\": \"The id of the the contract to be analyzed\"},\n {\"name\": \"title\", \"type\": str, \"desc\": \"The title of the the contract to be analyzed\"},\n {\"name\": \"contract\", \"type\": str, \"desc\": \"The content of the the contract to be analyzed\"},\n ]\n super().__init__(id=\"cuad\", schema=input_cols)\n\n # Load dataset from local files\n dataset = load_cuad_data(split=split)\n self.dataset = self._construct_dataset(dataset, num_contracts, seed)\n\n def _construct_dataset(self, dataset, num_contracts, seed: int=42):\n # get the set of unique contract titles; to ensure the order of the contracts is\n # preserved, we use a list rather than using python's set()\n contract_titles = []\n for row in dataset:\n if row[\"title\"] not in contract_titles:\n contract_titles.append(row[\"title\"])\n\n # shuffle the contracts for the given seed\n rng = np.random.default_rng(seed=seed)\n rng.shuffle(contract_titles)\n\n # get the first num_contracts\n contract_titles = contract_titles[:num_contracts]\n\n # construct the dataset one contract at a time\n new_dataset = []\n for title in contract_titles:\n # get the rows for this contract\n contract_rows = [row for row in dataset if row[\"title\"] == title]\n\n # construct the contract; we get the contract_id and contract text from the first row\n contract = {\n \"contract_id\": contract_rows[0][\"id\"],\n \"title\": title,\n \"contract\": contract_rows[0][\"context\"],\n }\n\n # add the rows to the dataset\n new_dataset.append(contract)\n\n return new_dataset\n\n def __len__(self):\n return self.num_contracts\n\n def __getitem__(self, idx: int):\n return self.dataset[idx]\n\n# Compute the precision and recall for the entire dataset.\n# Each row in the dataframes should correspond to a contract.\n# The columns should be the extracted fields (categories in CUAD_CATEGORIES).\ndef compute_precision_recall(label_df, preds_df):\n tp, fp, fn = 0, 0, 0\n\n label_df = label_df.sort_values(\"contract_id\").reset_index(drop=True)\n preds_df = preds_df.sort_values(\"contract_id\").reset_index(drop=True)\n\n assert label_df.shape == preds_df.shape, (\n f\"Label and prediction dataframes have different shapes, label shape: {label_df.shape} vs preds shape {preds_df.shape}\"\n )\n\n categories = [category[\"Category\"] for category in CUAD_CATEGORIES]\n\n for label_row, pred_row in zip(label_df.iterrows(), preds_df.iterrows()):\n assert label_row[1][\"contract_id\"] == pred_row[1][\"contract_id\"], (\n f\"IDs do not match. label id: {label_row[1]['contract_id']} vs pred id: {pred_row[1]['contract_id']}\"\n )\n for category in categories:\n substr_ok = \"Parties\" in category\n\n labels = label_row[1][category]\n assert isinstance(labels, list)\n\n preds = pred_row[1][category]\n preds = handle_empty_preds(preds)\n\n entry_tp, entry_fp, entry_fn = evaluate_entry(labels, preds, substr_ok)\n tp += entry_tp\n fp += entry_fp\n fn += entry_fn\n\n precision = tp / (tp + fp) if tp + fp > 0 else np.nan\n recall = tp / (tp + fn) if tp + fn > 0 else np.nan\n\n return precision, recall\n\n\ndef parse_arguments():\n parser = argparse.ArgumentParser(description=\"Run CUAD demo\")\n parser.add_argument(\"--mode\", type=str, help=\"one-convert or separate-converts\", default=\"one-convert\")\n parser.add_argument(\"--test\", type=str, help=\"test time compute active or inactive\", default=\"active\")\n parser.add_argument(\"--constrained\", default=False, action=\"store_true\", help=\"Use constrained objective\")\n parser.add_argument(\n \"--sentinel-execution-strategy\",\n default=\"mab\",\n type=str,\n help=\"The engine to use. One of mab or random\",\n )\n parser.add_argument(\n \"--optimizer-strategy\",\n default=\"pareto\",\n type=str,\n help=\"The optimizer to use. One of pareto or greedy\",\n )\n parser.add_argument(\"--verbose\", default=False, action=\"store_true\", help=\"Print verbose output\")\n parser.add_argument(\n \"--seed\",\n default=42,\n type=int,\n help=\"Seed used to initialize RNG for MAB sampling algorithm\",\n )\n parser.add_argument(\n \"--k\",\n default=10,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--j\",\n default=3,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--sample-budget\",\n default=100,\n type=int,\n help=\"Total sample budget in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--exp-name\",\n default=None,\n type=str,\n help=\"The experiment name.\",\n )\n parser.add_argument(\n \"--priors-file\",\n default=None,\n type=str,\n help=\"A file with a dictionary mapping physical operator ids to prior belief on their performance\",\n )\n parser.add_argument(\n \"--cost\",\n default=1.0,\n type=float,\n help=\"The cost budget for the optimization\",\n )\n return parser.parse_args()\n\n\ndef build_cuad_query(dataset, mode):\n assert mode in [\"one-convert\", \"separate-converts\"]\n\n if mode == \"one-convert\":\n cols = []\n for category in CUAD_CATEGORIES:\n desc = (\n f\"Extract the text spans (if they exist) from the contract corresponding to {category['Description']}\"\n )\n cols.append({\"name\": category[\"Category\"], \"type\": list[str], \"desc\": desc})\n\n desc = \"Extract the text spans (if they exist) from the contract.\"\n dataset = dataset.sem_map(cols, desc=desc, depends_on=[\"contract\"])\n elif mode == \"separate-converts\":\n for category in CUAD_CATEGORIES:\n desc = (\n f\"Extract the text spans (if they exist) from the contract corresponding to {category['Description']}\"\n )\n dataset = dataset.sem_map(\n [{\"name\": category[\"Category\"], \"type\": list[str], \"desc\": desc}],\n desc=category[\"Description\"],\n depends_on=[\"contract\"],\n )\n\n return dataset\n\n\ndef main():\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n args = parse_arguments()\n\n # create directory for profiling data\n os.makedirs(\"max-quality-at-cost-data\", exist_ok=True)\n\n # create validator for CUAD\n validator = CUADValidator(num_contracts=25, seed=args.seed)\n\n # create datasets for CUAD\n dataset = CUADDataset(split=\"test\", num_contracts=100, seed=args.seed)\n train_dataset = CUADDataset(split=\"train\", num_contracts=25, seed=args.seed)\n train_dataset = {train_dataset.id: train_dataset}\n print(\"Created datasets\")\n\n # Build and run the CUAD query\n query = build_cuad_query(dataset, args.mode)\n print(\"Built query; Starting query execution\")\n\n # set the optimization policy; constraint set to 25% percentile from unconstrained plans\n policy = MaxQualityAtFixedCost(max_cost=args.cost) if args.cost < 999 else MaxQuality()\n print(f\"USING POLICY: {policy}\")\n\n sentinel_strategy = args.sentinel_execution_strategy\n optimizer_strategy = args.optimizer_strategy\n seed = args.seed\n k = args.k\n j = args.j\n sample_budget = args.sample_budget\n exp_name = (\n f\"cuad-strategy-{optimizer_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}\"\n if args.exp_name is None\n else args.exp_name\n )\n priors = None\n if args.priors_file is not None:\n with open(args.priors_file) as f:\n priors = json.load(f)\n\n config = pz.QueryProcessorConfig(\n policy=policy,\n verbose=False,\n optimizer_strategy=optimizer_strategy,\n sentinel_execution_strategy=sentinel_strategy,\n execution_strategy=\"parallel\",\n max_workers=64,\n available_models=[\n Model.GPT_4o,\n Model.GPT_4o_MINI,\n Model.LLAMA3_1_8B,\n Model.LLAMA3_3_70B,\n # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments`\n Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B,\n ],\n allow_bonded_query=True,\n allow_critic=True,\n allow_mixtures=True,\n allow_rag_reduction=True,\n progress=True,\n seed=seed,\n k=k,\n j=j,\n sample_budget=sample_budget,\n exp_name=exp_name,\n priors=priors,\n )\n\n print(f\"EXPERIMENT NAME: {exp_name}\")\n data_record_collection = query.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator)\n print(\"Query execution completed\")\n\n # save statistics\n execution_stats_dict = data_record_collection.execution_stats.to_json()\n with open(f\"max-quality-at-cost-data/{exp_name}-stats.json\", \"w\") as f:\n json.dump(execution_stats_dict, f)\n\n pred_df = data_record_collection.to_df()\n label_df = get_label_df(num_contracts=100, seed=args.seed)\n # pred_df.to_csv(f\"{exp_name}-pred.csv\", index=False)\n # label_df.to_csv(f\"{exp_name}-label.csv\", index=False)\n final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0]\n final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id]\n\n prec, recall = compute_precision_recall(label_df, pred_df)\n f1 = 2 * (prec * recall) / (prec + recall) if prec + recall > 0 else 0.0\n stats_dict = {\n \"precision\": prec,\n \"recall\": recall,\n \"f1\": f1,\n \"optimization_time\": data_record_collection.execution_stats.optimization_time,\n \"optimization_cost\": data_record_collection.execution_stats.optimization_cost,\n \"plan_execution_time\": data_record_collection.execution_stats.plan_execution_time,\n \"plan_execution_cost\": data_record_collection.execution_stats.plan_execution_cost,\n \"total_execution_time\": data_record_collection.execution_stats.total_execution_time,\n \"total_execution_cost\": data_record_collection.execution_stats.total_execution_cost,\n \"plan_str\": final_plan_str,\n }\n with open(f\"max-quality-at-cost-data/{exp_name}-metrics.json\", \"w\") as f:\n json.dump(stats_dict, f)\n\n print(f\"Precision: {prec:.3f}, Recall: {recall:.3f}, F1: {f1:.3f}\")\n print(f\"Optimization time: {data_record_collection.execution_stats.optimization_time}\")\n print(f\"Optimization cost: {data_record_collection.execution_stats.optimization_cost}\")\n print(f\"Plan Exec. time: {data_record_collection.execution_stats.plan_execution_time}\")\n print(f\"Plan Exec. cost: {data_record_collection.execution_stats.plan_execution_cost}\")\n print(f\"Total Execution time: {data_record_collection.execution_stats.total_execution_time}\")\n print(f\"Total Execution Cost: {data_record_collection.execution_stats.total_execution_cost}\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "abacus-research/cuad-priors.json", "content": "{\"00c93aec22\": {\"quality\": 0.5304878048780488, \"cost\": 0.01609626, \"time\": 121.91315126419067}, \"00f4acd0d3\": {\"quality\": 0.6195121951219512, \"cost\": 0.007308299999999999, \"time\": 62.89620461463928}, \"0121878170\": {\"quality\": 0.7871196283391406, \"cost\": 0.084214564, \"time\": 137.42955293655396}, \"01c2f973ad\": {\"quality\": 0.6369918699186992, \"cost\": 0.00766284, \"time\": 88.04302344322204}, \"01fca3c717\": {\"quality\": 0.6376887340301974, \"cost\": 0.067807144, \"time\": 56.765127992630006}, \"02078988c1\": {\"quality\": 0.6195121951219512, \"cost\": 0.010957832, \"time\": 58.52112374305725}, \"021604dec1\": {\"quality\": 0.32056910569105695, \"cost\": 0.014393556000000002, \"time\": 102.96775641441346}, \"0262668df7\": {\"quality\": 0.7521835075493613, \"cost\": 0.015505927999999999, \"time\": 110.59786329269409}, \"0267c97b70\": {\"quality\": 0.6195121951219512, \"cost\": 0.007206299999999999, \"time\": 78.4086401939392}, \"02d6cdecdc\": {\"quality\": 0.17902439024390246, \"cost\": 0.043331628, \"time\": 118.95345692634582}, \"033ca325e6\": {\"quality\": 0.6308943089430895, \"cost\": 0.006504208, \"time\": 47.695338296890256}, \"0364b5e990\": {\"quality\": 0.6195121951219512, \"cost\": 0.069325568, \"time\": 129.42635221481322}, \"0375ea52c9\": {\"quality\": 0.6893495934959348, \"cost\": 0.026699588000000003, \"time\": 43.96490340232849}, \"038a5f0a62\": {\"quality\": 0.6491056910569106, \"cost\": 0.01088205, \"time\": 142.37771649360656}, \"039803b3b1\": {\"quality\": 0.8145180023228804, \"cost\": 0.06754355, \"time\": 77.90279107093811}, \"03b972cb56\": {\"quality\": 0.20228803716608595, \"cost\": 0.04939144400000001, \"time\": 117.83176441192627}, \"042d933706\": {\"quality\": 0.6789430894308943, \"cost\": 0.011810856, \"time\": 96.84674339294433}, \"050b21ce37\": {\"quality\": 0.6331707317073171, \"cost\": 0.063467788, \"time\": 138.06499099731445}, \"0524f42520\": {\"quality\": 0.5098606271777004, \"cost\": 0.100158564, \"time\": 119.39450807571411}, \"05420351e5\": {\"quality\": 0.28818815331010456, \"cost\": 0.03700374, \"time\": 99.57810640335083}, \"057e332ab1\": {\"quality\": 0.5822299651567945, \"cost\": 0.071891988, \"time\": 137.52664232254028}, \"0646f3f0fb\": {\"quality\": 0.4296747967479675, \"cost\": 0.06800707800000001, \"time\": 122.81284818649291}, \"06493715cc\": {\"quality\": 0.6146341463414634, \"cost\": 0.009369455999999998, \"time\": 101.42031364440918}, \"0659531b94\": {\"quality\": 0.7977235772357723, \"cost\": 0.06570914000000001, \"time\": 88.527423286438}, \"067ee6e91b\": {\"quality\": 0.7303135888501743, \"cost\": 0.012997799999999999, \"time\": 91.4061038017273}, \"0695f9b5fc\": {\"quality\": 0.6195121951219512, \"cost\": 0.012743448, \"time\": 123.27798008918762}, \"06e94a0f2e\": {\"quality\": 0.6381300813008131, \"cost\": 0.022528364000000002, \"time\": 70.015203332901}, \"073ef31d23\": {\"quality\": 0.6711730545876887, \"cost\": 0.06321265200000001, \"time\": 125.18851342201233}, \"078a7e545e\": {\"quality\": 0.5136469221835075, \"cost\": 0.06231631000000001, \"time\": 139.78958024978638}, \"079feb14a8\": {\"quality\": 0.5636585365853659, \"cost\": 0.012793146, \"time\": 83.61773014068604}, \"07a3a7daf7\": {\"quality\": 0.6195121951219512, \"cost\": 0.00968809, \"time\": 64.83957290649414}, \"08127cd6dd\": {\"quality\": 0.6773170731707316, \"cost\": 0.018527102, \"time\": 74.52640342712402}, \"0833133620\": {\"quality\": 0.783089430894309, \"cost\": 0.011579322, \"time\": 103.93566818237305}, \"087a2cabc4\": {\"quality\": 0.7773054587688735, \"cost\": 0.041329712000000005, \"time\": 149.253262424469}, \"08bf8cc191\": {\"quality\": 0.6195121951219512, \"cost\": 0.018227304, \"time\": 146.171551656723}, \"08e1802287\": {\"quality\": 0.40414634146341466, \"cost\": 0.061857968, \"time\": 639.4221799850463}, \"08f7f63b30\": {\"quality\": 0.6039024390243902, \"cost\": 0.06393643200000002, \"time\": 150.91687684059144}, \"090cd3ef31\": {\"quality\": 0.5126016260162601, \"cost\": 0.024500464, \"time\": 242.8270983695984}, \"0947216ece\": {\"quality\": 0.6116260162601627, \"cost\": 0.022374364, \"time\": 86.5027738571167}, \"096d51f670\": {\"quality\": 0.6195121951219512, \"cost\": 0.015167303999999998, \"time\": 168.75218114852905}, \"09791c731b\": {\"quality\": 0.6195121951219512, \"cost\": 0.010591104, \"time\": 98.68934841156006}, \"0990c0d4f8\": {\"quality\": 0.7328455284552845, \"cost\": 0.015548904000000002, \"time\": 126.22301592826844}, \"0a4c1bbb4a\": {\"quality\": 0.6195121951219512, \"cost\": 0.01167957, \"time\": 112.43891968727112}, \"0ac969dde3\": {\"quality\": 0.532520325203252, \"cost\": 0.019075388000000002, \"time\": 182.81917290687562}, \"0b4ab72197\": {\"quality\": 0.6195121951219512, \"cost\": 0.01298358, \"time\": 88.19813923835754}, \"0bf9d31691\": {\"quality\": 0.6195121951219512, \"cost\": 0.06748344, \"time\": 112.7701108455658}, \"0c020b86a3\": {\"quality\": 0.48138211382113827, \"cost\": 0.012613356000000003, \"time\": 94.54495902061463}, \"0c6c7fe96a\": {\"quality\": 0.6439024390243903, \"cost\": 0.017969835999999996, \"time\": 175.61341876983641}, \"0c81c8996a\": {\"quality\": 0.6195121951219512, \"cost\": 0.008283972, \"time\": 123.77774615287781}, \"0cd25da9fe\": {\"quality\": 0.41504065040650406, \"cost\": 0.071906784, \"time\": 151.1712607383728}, \"0cd78f33d8\": {\"quality\": 0.6969337979094077, \"cost\": 0.12822476, \"time\": 110.73728203773499}, \"0cdc5954dd\": {\"quality\": 0.39105691056910574, \"cost\": 0.013535116, \"time\": 110.69660997390747}, \"0d53dd53c1\": {\"quality\": 0.4902439024390244, \"cost\": 0.08892936000000001, \"time\": 129.61446180343628}, \"0d8436af32\": {\"quality\": 0.6047386759581882, \"cost\": 0.03048576, \"time\": 119.35665726661682}, \"0e38896654\": {\"quality\": 0.7874680603948896, \"cost\": 0.029542839999999997, \"time\": 118.93085074424744}, \"0ec672e7c8\": {\"quality\": 0.3153774680603949, \"cost\": 0.018613362, \"time\": 156.979243183136}, \"0ed243f788\": {\"quality\": 0.6195121951219512, \"cost\": 0.014807712, \"time\": 171.58208327293397}, \"0eeb372802\": {\"quality\": 0.6195121951219512, \"cost\": 0.017926556, \"time\": 135.83013033866882}, \"0ef0becc1b\": {\"quality\": 0.3480371660859466, \"cost\": 0.058053114, \"time\": 112.96433029174804}, \"0fefead197\": {\"quality\": 0.7968176538908246, \"cost\": 0.09321153600000001, \"time\": 177.5354432106018}, \"0ff126ebf8\": {\"quality\": 0.3920325203252033, \"cost\": 0.01680763, \"time\": 116.37514338493347}, \"10d1d4bdeb\": {\"quality\": 0.6195121951219512, \"cost\": 0.066024524, \"time\": 129.0803412914276}, \"114a097c53\": {\"quality\": 0.6195121951219512, \"cost\": 0.00815448, \"time\": 68.65160498619079}, \"1175ee37e6\": {\"quality\": 0.5414634146341463, \"cost\": 0.002796924, \"time\": 66.07583198547363}, \"11bc996d48\": {\"quality\": 0.7575261324041811, \"cost\": 0.011037809999999999, \"time\": 160.15645961761476}, \"11ded03305\": {\"quality\": 0.7726016260162601, \"cost\": 0.07187866600000001, \"time\": 108.60114121437073}, \"123fb650fb\": {\"quality\": 0.6260162601626017, \"cost\": 0.012730068, \"time\": 158.32172050476075}, \"132f6f3946\": {\"quality\": 0.6478048780487804, \"cost\": 0.06792896600000001, \"time\": 142.56146936416627}, \"133ee5023f\": {\"quality\": 0.6323577235772359, \"cost\": 0.009341408, \"time\": 76.29307880401612}, \"13a009fe0c\": {\"quality\": 0.6134146341463416, \"cost\": 0.020489088, \"time\": 159.91688990592957}, \"13da306f84\": {\"quality\": 0.504390243902439, \"cost\": 0.013949744, \"time\": 125.56137013435364}, \"13e717e41e\": {\"quality\": 0.5087804878048781, \"cost\": 0.055634039999999996, \"time\": 89.58166871070861}, \"13f75f9bd0\": {\"quality\": 0.3702439024390244, \"cost\": 0.009800003999999998, \"time\": 140.3795708656311}, \"140ededb41\": {\"quality\": 0.7700813008130082, \"cost\": 0.0035910299999999994, \"time\": 54.85756406784058}, \"142e59c03f\": {\"quality\": 0.39747967479674795, \"cost\": 0.080664288, \"time\": 125.91337852478027}, \"142f3a7c70\": {\"quality\": 0.6195121951219512, \"cost\": 0.009178488000000002, \"time\": 142.1152557849884}, \"1468dddecc\": {\"quality\": 0.6699186991869919, \"cost\": 0.007540319999999999, \"time\": 112.98228130340576}, \"15af009a01\": {\"quality\": 0.47317073170731705, \"cost\": 0.07114547600000001, \"time\": 165.90874967575073}, \"15b80a55d3\": {\"quality\": 0.5414634146341464, \"cost\": 0.06812305999999999, \"time\": 156.45735163688659}, \"1625e624c5\": {\"quality\": 0.56931475029036, \"cost\": 0.05795948000000001, \"time\": 88.15128273963929}, \"16cff1c1e9\": {\"quality\": 0.6195121951219512, \"cost\": 0.06384869200000001, \"time\": 149.38432698249818}, \"17407df027\": {\"quality\": 0.5050406504065041, \"cost\": 0.06935714000000001, \"time\": 157.59272437095643}, \"176da24f53\": {\"quality\": 0.6195121951219512, \"cost\": 0.006307396000000002, \"time\": 48.773886156082156}, \"179379555f\": {\"quality\": 0.43966318234610924, \"cost\": 0.008765568, \"time\": 131.12133555412294}, \"181c91d1be\": {\"quality\": 0.4508943089430894, \"cost\": 0.016041972, \"time\": 104.77644038200378}, \"183743e76e\": {\"quality\": 0.6195121951219512, \"cost\": 0.007336812, \"time\": 88.6048484802246}, \"186b58c209\": {\"quality\": 0.47317073170731705, \"cost\": 0.064688372, \"time\": 136.70242128372192}, \"187eace9fe\": {\"quality\": 0.6195121951219512, \"cost\": 0.061498704, \"time\": 111.12849621772766}, \"190ed2e1b6\": {\"quality\": 0.4953193960511033, \"cost\": 0.06357867000000002, \"time\": 139.53685355186462}, \"191aafe1a6\": {\"quality\": 0.6285017421602788, \"cost\": 0.05959545200000001, \"time\": 133.747340965271}, \"194919ad28\": {\"quality\": 0.6195121951219512, \"cost\": 0.061897668, \"time\": 120.82026715278626}, \"197bb53f10\": {\"quality\": 0.4398373983739837, \"cost\": 0.012262326, \"time\": 128.21201944351196}, \"19ba7d0617\": {\"quality\": 0.8144715447154471, \"cost\": 0.18371900000000002, \"time\": 127.1558424949646}, \"19e3db7fe7\": {\"quality\": 0.6195121951219512, \"cost\": 0.016522608, \"time\": 157.01711511611938}, \"1a08cb3f50\": {\"quality\": 0.6195121951219512, \"cost\": 0.06344755, \"time\": 113.1626263141632}, \"1a169179f6\": {\"quality\": 0.3619976771196284, \"cost\": 0.096862584, \"time\": 173.04007925987244}, \"1a71d61ac4\": {\"quality\": 0.6055284552845528, \"cost\": 0.044680528000000004, \"time\": 134.11820168495177}, \"1ad856985f\": {\"quality\": 0.6195121951219512, \"cost\": 0.015116656000000001, \"time\": 213.2139684677124}, \"1adec2dca2\": {\"quality\": 0.6195121951219512, \"cost\": 0.018743704, \"time\": 189.0085569858551}, \"1b04a2b184\": {\"quality\": 0.27695702671312433, \"cost\": 0.018901547999999997, \"time\": 190.3927951812744}, \"1b28439bd7\": {\"quality\": 0.595040650406504, \"cost\": 0.018163196000000003, \"time\": 130.19455728530883}, \"1b2c667b15\": {\"quality\": 0.6146341463414634, \"cost\": 0.06703916, \"time\": 141.4092381477356}, \"1b4511eada\": {\"quality\": 0.5601277584204414, \"cost\": 0.06213515000000001, \"time\": 197.098247051239}, \"1b7e6cad66\": {\"quality\": 0.6678048780487805, \"cost\": 0.061792784, \"time\": 95.13608679771423}, \"1beb2fac62\": {\"quality\": 0.5772357723577235, \"cost\": 0.023681768000000002, \"time\": 99.76408967971801}, \"1c347e4d91\": {\"quality\": 0.5130894308943089, \"cost\": 0.005374038, \"time\": 119.8269275188446}, \"1c35bf4be6\": {\"quality\": 0.6613124274099883, \"cost\": 0.06615356000000001, \"time\": 178.00547165870665}, \"1c4bbf8f7e\": {\"quality\": 0.812137049941928, \"cost\": 0.09307272000000003, \"time\": 165.35304036140442}, \"1c5f1341f6\": {\"quality\": 0.6847967479674797, \"cost\": 0.035318894, \"time\": 106.12908926010132}, \"1c71804bec\": {\"quality\": 0.5271544715447154, \"cost\": 0.037727392, \"time\": 100.24818325042725}, \"1cc6d9efb6\": {\"quality\": 0.4643902439024391, \"cost\": 0.018275512, \"time\": 88.59051547050476}, \"1ce3d77039\": {\"quality\": 0.6304878048780488, \"cost\": 0.0014104099999999997, \"time\": 62.92718415260315}, \"1d26090364\": {\"quality\": 0.5578048780487805, \"cost\": 0.00619749, \"time\": 73.83342995643616}, \"1d87f97e62\": {\"quality\": 0.6894192799070848, \"cost\": 0.013183738, \"time\": 122.42380046844482}, \"1d90fb8ca6\": {\"quality\": 0.7902903600464576, \"cost\": 0.08424590000000001, \"time\": 134.87732191085814}, \"1da2369719\": {\"quality\": 0.676492450638792, \"cost\": 0.010263576, \"time\": 102.77953281402588}, \"1e18e60895\": {\"quality\": 0.6342276422764228, \"cost\": 0.0014448139999999998, \"time\": 62.875070858001706}, \"1e1bf7e88b\": {\"quality\": 0.38325203252032525, \"cost\": 0.01685045, \"time\": 99.01278185844421}, \"1e8b3521f8\": {\"quality\": 0.6149709639953542, \"cost\": 0.006865926, \"time\": 112.37140879631042}, \"1f412964ff\": {\"quality\": 0.6195121951219512, \"cost\": 0.059373496000000005, \"time\": 141.1672589302063}, \"1f72cfb78a\": {\"quality\": 0.2634494773519164, \"cost\": 0.04392928, \"time\": 141.55319528579713}, \"1fb5d170ad\": {\"quality\": 0.4796399535423926, \"cost\": 0.061984526000000005, \"time\": 130.7815794467926}, \"20180dd292\": {\"quality\": 0.6195121951219512, \"cost\": 0.06752007200000001, \"time\": 180.62230010032653}, \"2018bef45f\": {\"quality\": 0.624390243902439, \"cost\": 0.010508668, \"time\": 100.28150401115417}, \"2075ff1d04\": {\"quality\": 0.3623577235772358, \"cost\": 0.015088508000000004, \"time\": 1278.50945892334}, \"208a98f514\": {\"quality\": 0.775156794425087, \"cost\": 0.017165512, \"time\": 166.07545523643495}, \"20904e5c14\": {\"quality\": 0.476260162601626, \"cost\": 0.072447476, \"time\": 98.36213431358337}, \"20afc3d539\": {\"quality\": 0.5903716608594658, \"cost\": 0.084462272, \"time\": 125.81502628326416}, \"20e10af7d4\": {\"quality\": 0.6369918699186992, \"cost\": 0.011088839999999999, \"time\": 154.89131064414977}, \"20e2c0b057\": {\"quality\": 0.7137398373983739, \"cost\": 0.014822027999999998, \"time\": 160.84442710876465}, \"211b89b4cd\": {\"quality\": 0.3874680603948897, \"cost\": 0.0074940599999999994, \"time\": 110.43994216918945}, \"21386082aa\": {\"quality\": 0.44699186991869916, \"cost\": 0.04453856, \"time\": 132.42854318618774}, \"21b2b8ebd1\": {\"quality\": 0.4067479674796748, \"cost\": 0.010719648000000002, \"time\": 121.89121770858765}, \"21b78249a7\": {\"quality\": 0.40636469221835075, \"cost\": 0.08786833200000001, \"time\": 114.88702239990235}, \"21bed16a7d\": {\"quality\": 0.42804878048780487, \"cost\": 0.021284448, \"time\": 757.0631816387177}, \"2200d969d0\": {\"quality\": 0.729616724738676, \"cost\": 0.103071816, \"time\": 184.93275413513183}, \"220d008704\": {\"quality\": 0.6195121951219512, \"cost\": 0.061679156, \"time\": 194.95589275360106}, \"2251d21392\": {\"quality\": 0.3285365853658536, \"cost\": 0.063983622, \"time\": 125.72053866386413}, \"23566f15ab\": {\"quality\": 0.6269918699186992, \"cost\": 0.0011016140000000001, \"time\": 68.13801875114441}, \"23a9506d36\": {\"quality\": 0.6296747967479674, \"cost\": 0.04247636400000001, \"time\": 287.74875631332395}, \"24957f3a43\": {\"quality\": 0.36911730545876886, \"cost\": 0.007789104, \"time\": 66.6147982120514}, \"2529e2f8b0\": {\"quality\": 0.6348780487804878, \"cost\": 0.018974088000000004, \"time\": 46.99674005508423}, \"252f01ac5b\": {\"quality\": 0.7063414634146341, \"cost\": 0.09021934000000001, \"time\": 138.44833331108094}, \"25fadf0883\": {\"quality\": 0.6009756097560977, \"cost\": 0.069262768, \"time\": 195.04961967468262}, \"2609bfd616\": {\"quality\": 0.4904878048780488, \"cost\": 0.014383092, \"time\": 138.67839546203612}, \"2629f3e324\": {\"quality\": 0.3373867595818815, \"cost\": 0.06496914000000001, \"time\": 199.1607180118561}, \"262e4298f9\": {\"quality\": 0.5845528455284553, \"cost\": 0.06646106, \"time\": 168.6952573776245}, \"26cc40d3bb\": {\"quality\": 0.6195121951219512, \"cost\": 0.066186748, \"time\": 191.06331505775452}, \"2728c8eb6a\": {\"quality\": 0.6195121951219512, \"cost\": 0.013364808, \"time\": 136.0991479873657}, \"27bc52befa\": {\"quality\": 0.7133333333333334, \"cost\": 0.035852640000000005, \"time\": 102.54774498939514}, \"27daa50458\": {\"quality\": 0.5414634146341464, \"cost\": 0.008815032, \"time\": 95.5854898929596}, \"2821795e69\": {\"quality\": 0.47317073170731716, \"cost\": 0.06960079, \"time\": 199.99654774665834}, \"28369b2421\": {\"quality\": 0.5271544715447154, \"cost\": 0.012585374, \"time\": 139.39593086242675}, \"28421e6d62\": {\"quality\": 0.6195121951219512, \"cost\": 0.010284108, \"time\": 126.62169485092163}, \"2936c3e43e\": {\"quality\": 0.7311382113821139, \"cost\": 0.022435948, \"time\": 190.63684725761414}, \"293ec5edca\": {\"quality\": 0.8191753774680605, \"cost\": 0.014760236, \"time\": 130.9795027732849}, \"29409d0894\": {\"quality\": 0.30975609756097555, \"cost\": 0.023374712000000002, \"time\": 67.99601030349731}, \"294258298a\": {\"quality\": 0.6918466898954704, \"cost\": 0.060563396000000005, \"time\": 171.09867010116577}, \"294e541235\": {\"quality\": 0.6195121951219512, \"cost\": 0.004267548, \"time\": 63.70019145011902}, \"295ed5e759\": {\"quality\": 0.2658536585365854, \"cost\": 0.00976596, \"time\": 102.0877944469452}, \"2960431101\": {\"quality\": 0.40162601626016264, \"cost\": 0.010855686, \"time\": 106.73301725387573}, \"29892d8468\": {\"quality\": 0.6195121951219512, \"cost\": 0.016309508, \"time\": 152.2814826965332}, \"299a0aeb65\": {\"quality\": 0.6601509872241579, \"cost\": 0.014925237999999999, \"time\": 216.44449853897095}, \"29ad99e3ed\": {\"quality\": 0.6195121951219512, \"cost\": 0.06017612800000001, \"time\": 140.98456511497497}, \"2a5edac2de\": {\"quality\": 0.4360046457607433, \"cost\": 0.093795708, \"time\": 186.12790670394898}, \"2a7d15f4a7\": {\"quality\": 0.6637398373983739, \"cost\": 0.0017069580000000002, \"time\": 66.5458746433258}, \"2aa996de6a\": {\"quality\": 0.40952380952380957, \"cost\": 0.09028132800000001, \"time\": 171.12979340553284}, \"2ac4fb293f\": {\"quality\": 0.6234146341463415, \"cost\": 0.039522680000000004, \"time\": 134.96180925369262}, \"2afeff0083\": {\"quality\": 0.5430894308943089, \"cost\": 0.065989844, \"time\": 233.06356620788574}, \"2b2bc9568b\": {\"quality\": 0.6195121951219512, \"cost\": 0.071159096, \"time\": 242.5479420185089}, \"2b5679d248\": {\"quality\": 0.6195121951219512, \"cost\": 0.018697384, \"time\": 247.21016097068787}, \"2bcf54cda1\": {\"quality\": 0.5373983739837398, \"cost\": 0.06900906000000001, \"time\": 200.87765913009645}, \"2bd39ee744\": {\"quality\": 0.7727642276422764, \"cost\": 0.029763757999999994, \"time\": 142.61127648353576}, \"2bf38d797f\": {\"quality\": 0.6145528455284553, \"cost\": 0.015432928000000002, \"time\": 154.4491545200348}, \"2c4f4f304e\": {\"quality\": 0.5686875725900116, \"cost\": 0.02265774, \"time\": 104.49429354667663}, \"2c5cf9eb26\": {\"quality\": 0.6195121951219512, \"cost\": 0.010875623999999999, \"time\": 208.2448058128357}, \"2c9a9f94c4\": {\"quality\": 0.6195121951219512, \"cost\": 0.016278144, \"time\": 211.14152693748474}, \"2d3bbc2d23\": {\"quality\": 0.5347386759581882, \"cost\": 0.011645886, \"time\": 142.9862838745117}, \"2de113167b\": {\"quality\": 0.3548780487804878, \"cost\": 0.01615302, \"time\": 215.62943921089172}, \"2de3eb2c19\": {\"quality\": 0.6495934959349594, \"cost\": 0.012367168000000001, \"time\": 84.4677538394928}, \"2e30394ac6\": {\"quality\": 0.42853658536585365, \"cost\": 0.013055616, \"time\": 162.57028393745424}, \"2e9c5cc9bf\": {\"quality\": 0.6195121951219512, \"cost\": 0.008993688000000001, \"time\": 123.94604053497315}, \"2f1573da80\": {\"quality\": 0.628048780487805, \"cost\": 0.012240792, \"time\": 185.1746757030487}, \"2f39d78f34\": {\"quality\": 0.6814169570267131, \"cost\": 0.08897421200000001, \"time\": 171.90278725624086}, \"2fc0cb3592\": {\"quality\": 0.3300813008130082, \"cost\": 0.011238624, \"time\": 162.5598623752594}, \"2fd9cd426a\": {\"quality\": 0.3750406504065041, \"cost\": 0.01883974, \"time\": 179.53315086364745}, \"300924ebae\": {\"quality\": 0.2581533101045296, \"cost\": 0.08039779600000001, \"time\": 140.4485348701477}, \"3019af79b3\": {\"quality\": 0.5535423925667828, \"cost\": 0.014683848, \"time\": 1308.0872743606567}, \"302c1d97fc\": {\"quality\": 0.6195121951219512, \"cost\": 0.01627172, \"time\": 136.45579180717468}, \"30ae4cbe91\": {\"quality\": 0.37552845528455286, \"cost\": 0.006890768, \"time\": 750.692762708664}, \"30c1f9ddf1\": {\"quality\": 0.6262369337979093, \"cost\": 0.071551416, \"time\": 112.11581921577454}, \"30cd375570\": {\"quality\": 0.6195121951219512, \"cost\": 0.06602648800000002, \"time\": 118.63089265823365}, \"3169782cbb\": {\"quality\": 0.4402439024390244, \"cost\": 0.060568003999999995, \"time\": 103.64021511077881}, \"3172fc459a\": {\"quality\": 0.4475609756097561, \"cost\": 0.073885504, \"time\": 161.98054652214051}, \"318499c14b\": {\"quality\": 0.4873054587688733, \"cost\": 0.06648116000000001, \"time\": 187.55820865631102}, \"31a32be94d\": {\"quality\": 0.8294541231126598, \"cost\": 0.08899002, \"time\": 188.3829703807831}, \"32b101d807\": {\"quality\": 0.6408594657375145, \"cost\": 0.023361424, \"time\": 210.3520890712738}, \"32e2c7ad7f\": {\"quality\": 0.317479674796748, \"cost\": 0.06465559000000001, \"time\": 176.4482195854187}, \"33459cd29c\": {\"quality\": 0.6195121951219512, \"cost\": 0.014899176000000004, \"time\": 156.23946170806886}, \"33a187e74f\": {\"quality\": 0.6709756097560975, \"cost\": 0.01123047, \"time\": 201.9312825202942}, \"33bab4f766\": {\"quality\": 0.7291869918699188, \"cost\": 0.063704096, \"time\": 215.9541217803955}, \"34922140da\": {\"quality\": 0.5666666666666667, \"cost\": 0.06817234800000001, \"time\": 192.9766547679901}, \"3511b5e1d0\": {\"quality\": 0.6195121951219512, \"cost\": 0.013343184, \"time\": 129.7560082912445}, \"3513311c2d\": {\"quality\": 0.48170731707317077, \"cost\": 0.064084452, \"time\": 141.37073793411255}, \"3513e54767\": {\"quality\": 0.6195121951219512, \"cost\": 0.010438112, \"time\": 88.52075481414795}, \"353f0cb1ac\": {\"quality\": 0.6175609756097561, \"cost\": 0.006175324, \"time\": 85.6404734134674}, \"3550bf88cb\": {\"quality\": 0.6732752613240418, \"cost\": 0.012484512, \"time\": 185.91844930648804}, \"35610fb420\": {\"quality\": 0.7706271777003485, \"cost\": 0.009480828, \"time\": 151.24167275428772}, \"357267e14b\": {\"quality\": 0.40682926829268296, \"cost\": 0.017491424, \"time\": 768.3776504993439}, \"35baa5c3cc\": {\"quality\": 0.7191056910569106, \"cost\": 0.07052012, \"time\": 176.32511940002442}, \"3637084f91\": {\"quality\": 0.5414634146341464, \"cost\": 0.010371784, \"time\": 99.66692337989807}, \"368a497102\": {\"quality\": 0.5840650406504064, \"cost\": 0.039775324, \"time\": 107.50217499732972}, \"36c66671ee\": {\"quality\": 0.6195121951219512, \"cost\": 0.010492836, \"time\": 127.38456826210022}, \"37456cb002\": {\"quality\": 0.800801393728223, \"cost\": 0.036142828, \"time\": 157.18044147491455}, \"3746ea5c03\": {\"quality\": 0.45056910569105685, \"cost\": 0.03366273200000001, \"time\": 102.54732451438903}, \"375ed248fe\": {\"quality\": 0.33772357723577234, \"cost\": 0.013014735999999999, \"time\": 85.65126795768738}, \"377cdf9209\": {\"quality\": 0.7822648083623693, \"cost\": 0.036440860000000005, \"time\": 189.91213726997375}, \"37d4d0f214\": {\"quality\": 0.6195121951219512, \"cost\": 0.071572196, \"time\": 152.68034386634827}, \"37ece7217f\": {\"quality\": 0.47804878048780486, \"cost\": 0.05685350800000001, \"time\": 76.75442943572997}, \"38075bb01f\": {\"quality\": 0.6195121951219512, \"cost\": 0.010187640000000001, \"time\": 148.4516242980957}, \"3831d758b1\": {\"quality\": 0.6195121951219512, \"cost\": 0.063214564, \"time\": 129.3372209072113}, \"38567d6a43\": {\"quality\": 0.6195121951219512, \"cost\": 0.008923896, \"time\": 136.575146484375}, \"3875787727\": {\"quality\": 0.6195121951219512, \"cost\": 0.06695744000000001, \"time\": 189.5042601108551}, \"389c54cbca\": {\"quality\": 0.633739837398374, \"cost\": 0.010009344, \"time\": 94.1163019657135}, \"3980f20caa\": {\"quality\": 0.5065040650406505, \"cost\": 0.020949968, \"time\": 150.0350682735443}, \"3997a836bd\": {\"quality\": 0.7323577235772357, \"cost\": 0.069346092, \"time\": 181.2115756034851}, \"39ad76f8ce\": {\"quality\": 0.5396747967479676, \"cost\": 0.06732360800000001, \"time\": 117.92112016677856}, \"39c0b7c171\": {\"quality\": 0.4475609756097561, \"cost\": 0.07215601600000002, \"time\": 221.94310250282288}, \"39cd4ca402\": {\"quality\": 0.6195121951219512, \"cost\": 0.007245383999999999, \"time\": 139.3380250453949}, \"3a32c98a53\": {\"quality\": 0.7996399535423925, \"cost\": 0.066284192, \"time\": 151.66175842285156}, \"3ac7fa4e46\": {\"quality\": 0.6183972125435541, \"cost\": 0.09699730400000002, \"time\": 154.47008938789367}, \"3ad6dcf559\": {\"quality\": 0.6195121951219512, \"cost\": 0.071971364, \"time\": 124.51828866004944}, \"3ae0de8663\": {\"quality\": 0.2573170731707317, \"cost\": 0.043973896, \"time\": 145.02047486305236}, \"3b2e8075ea\": {\"quality\": 0.7909523809523809, \"cost\": 0.007301412, \"time\": 154.77439031600952}, \"3b3676521a\": {\"quality\": 0.7468989547038328, \"cost\": 0.025589519999999998, \"time\": 245.33448405265807}, \"3b57530a56\": {\"quality\": 0.7408943089430895, \"cost\": 0.009371704, \"time\": 124.3612250328064}, \"3b6fbfa11d\": {\"quality\": 0.4241695702671313, \"cost\": 0.03699336, \"time\": 136.01204528808594}, \"3b81215e7a\": {\"quality\": 0.7074099883855982, \"cost\": 0.069720068, \"time\": 194.1675964832306}, \"3b9f8045d7\": {\"quality\": 0.5806620209059233, \"cost\": 0.09104832400000001, \"time\": 150.99677429199218}, \"3c5857683c\": {\"quality\": 0.5024390243902439, \"cost\": 0.07306957600000001, \"time\": 161.98392362594603}, \"3cbab8082e\": {\"quality\": 0.6873751451800232, \"cost\": 0.014340899999999998, \"time\": 174.04833178520204}, \"3d21104666\": {\"quality\": 0.24378629500580723, \"cost\": 0.126616, \"time\": 80.24543118476868}, \"3d71c4dd2c\": {\"quality\": 0.6786062717770036, \"cost\": 0.010625964, \"time\": 140.9857957839966}, \"3d9e24215e\": {\"quality\": 0.3852845528455284, \"cost\": 0.028569556000000003, \"time\": 57.36162734031677}, \"3e7efee65a\": {\"quality\": 0.5565040650406504, \"cost\": 0.066932276, \"time\": 163.1447666168213}, \"3ed0ad20ed\": {\"quality\": 0.32991869918699185, \"cost\": 0.071716344, \"time\": 165.24500761032104}, \"3f1a58aec9\": {\"quality\": 0.533739837398374, \"cost\": 0.013489895999999998, \"time\": 331.1659511566162}, \"3f2b07cb78\": {\"quality\": 0.19126596980255517, \"cost\": 0.041601888, \"time\": 124.82682638168335}, \"3f3ef494b0\": {\"quality\": 0.6195121951219512, \"cost\": 0.0015848999999999998, \"time\": 51.92140054702759}, \"3f62c3fbfc\": {\"quality\": 0.19586527293844366, \"cost\": 0.0071477519999999985, \"time\": 118.16850218772888}, \"3f730d8bfe\": {\"quality\": 0.7800813008130081, \"cost\": 0.083953044, \"time\": 111.95527267456055}, \"3f8d2ee81f\": {\"quality\": 0.6478048780487805, \"cost\": 0.042814124, \"time\": 129.17971215248107}, \"40104c813f\": {\"quality\": 0.6195121951219512, \"cost\": 0.016629716, \"time\": 190.76988382339476}, \"403b05da2d\": {\"quality\": 0.6195121951219512, \"cost\": 0.013792372, \"time\": 143.9212529182434}, \"403f0726fa\": {\"quality\": 0.44464576074332174, \"cost\": 0.07387800000000001, \"time\": 62.18464155197144}, \"4098178354\": {\"quality\": 0.6802322880371661, \"cost\": 0.06734463600000001, \"time\": 214.58685207366943}, \"409ff67607\": {\"quality\": 0.6195121951219512, \"cost\": 0.0056238419999999996, \"time\": 99.12217197418212}, \"40b3b6642c\": {\"quality\": 0.4744831591173055, \"cost\": 0.06449838599999999, \"time\": 133.02915487289428}, \"412c065b83\": {\"quality\": 0.6195121951219512, \"cost\": 0.009992927999999998, \"time\": 171.42633333206177}, \"4191118787\": {\"quality\": 0.6195121951219512, \"cost\": 0.014096268, \"time\": 191.99022674560547}, \"41d5b97871\": {\"quality\": 0.6195121951219512, \"cost\": 0.003545856, \"time\": 92.14297785758973}, \"41d8845655\": {\"quality\": 0.7837282229965157, \"cost\": 0.021811158000000004, \"time\": 109.06256651878357}, \"41ee202cac\": {\"quality\": 0.5507317073170731, \"cost\": 0.014144588000000003, \"time\": 1669.106618309021}, \"41fe4aee55\": {\"quality\": 0.6195121951219512, \"cost\": 0.06379543600000001, \"time\": 147.32905130386354}, \"42430ea391\": {\"quality\": 0.39349593495934954, \"cost\": 0.073369736, \"time\": 190.46762056350707}, \"42ddd48341\": {\"quality\": 0.4267479674796748, \"cost\": 0.023891804, \"time\": 166.09094681739808}, \"42f1e19aa7\": {\"quality\": 0.2668060394889663, \"cost\": 0.10482261600000001, \"time\": 180.36561794281005}, \"430a2ab32f\": {\"quality\": 0.49148664343786297, \"cost\": 0.08972675600000002, \"time\": 123.45088205337524}, \"4339427ad8\": {\"quality\": 0.6195121951219512, \"cost\": 0.06641424400000001, \"time\": 176.55676898956298}, \"4361bc7ea7\": {\"quality\": 0.433472706155633, \"cost\": 0.018289859999999998, \"time\": 1338.038490152359}, \"43c3cf9cb8\": {\"quality\": 0.6104065040650406, \"cost\": 0.011720016, \"time\": 66.8409245967865}, \"43d24fb32a\": {\"quality\": 0.3804878048780488, \"cost\": 0.016379803999999998, \"time\": 121.97038044929505}, \"43e9b39e5c\": {\"quality\": 0.7054587688734031, \"cost\": 0.07738095200000002, \"time\": 196.7220965385437}, \"44d6af5523\": {\"quality\": 0.6940650406504065, \"cost\": 0.010872959999999997, \"time\": 159.60600094795228}, \"44f189d813\": {\"quality\": 0.7999303135888501, \"cost\": 0.038299172000000006, \"time\": 177.3291199207306}, \"450f45a187\": {\"quality\": 0.6195121951219512, \"cost\": 0.016291588000000003, \"time\": 72.91690697669983}, \"453d0a5097\": {\"quality\": 0.39195121951219514, \"cost\": 0.029831792000000003, \"time\": 120.01631045341492}, \"4547ef4c8e\": {\"quality\": 0.4353658536585366, \"cost\": 0.065468638, \"time\": 134.60169172286987}, \"461846a52d\": {\"quality\": 0.6195121951219512, \"cost\": 0.006258096, \"time\": 108.38983693122864}, \"462e6ff849\": {\"quality\": 0.4353658536585366, \"cost\": 0.016916924, \"time\": 133.2640913963318}, \"4630853d32\": {\"quality\": 0.6390243902439025, \"cost\": 0.01612172, \"time\": 131.57029628753662}, \"46475b9e75\": {\"quality\": 0.7159001161440186, \"cost\": 0.011104722, \"time\": 122.19585280418396}, \"46654a1f32\": {\"quality\": 0.6195121951219512, \"cost\": 0.016634112, \"time\": 77.13833050727844}, \"466a3036b2\": {\"quality\": 0.6498257839721253, \"cost\": 0.018471648, \"time\": 207.47669353485108}, \"466d4d16dd\": {\"quality\": 0.5902439024390244, \"cost\": 0.011955879999999999, \"time\": 118.00577793121337}, \"46ed68152d\": {\"quality\": 0.5575493612078979, \"cost\": 0.016878404, \"time\": 116.81743607521057}, \"476a12876c\": {\"quality\": 0.6195121951219512, \"cost\": 0.007925976, \"time\": 132.9657735824585}, \"4778401a7a\": {\"quality\": 0.6119047619047618, \"cost\": 0.036857304, \"time\": 163.85823788642884}, \"47f9115b26\": {\"quality\": 0.6195121951219512, \"cost\": 0.06548748400000001, \"time\": 177.22612433433534}, \"48043e2304\": {\"quality\": 0.4546341463414635, \"cost\": 0.013231672000000003, \"time\": 139.5205948829651}, \"487f30e740\": {\"quality\": 0.6455284552845528, \"cost\": 0.04562078000000001, \"time\": 155.9966438770294}, \"488645cbd9\": {\"quality\": 0.6195121951219512, \"cost\": 0.0027527399999999996, \"time\": 95.60826187133789}, \"48bf87f7fe\": {\"quality\": 0.4546922183507549, \"cost\": 0.095429448, \"time\": 165.5019425392151}, \"4909061216\": {\"quality\": 0.2585365853658536, \"cost\": 0.018523724, \"time\": 212.3366045475006}, \"49731b1ccd\": {\"quality\": 0.2308246225319396, \"cost\": 0.010751964, \"time\": 158.96827306747437}, \"49ad844bd2\": {\"quality\": 0.6195121951219512, \"cost\": 0.015603239999999999, \"time\": 223.71964612007142}, \"49ca727e49\": {\"quality\": 0.653739837398374, \"cost\": 0.0016870099999999999, \"time\": 64.0728322505951}, \"4a23d8eff7\": {\"quality\": 0.6195121951219512, \"cost\": 0.06434071200000001, \"time\": 202.87527737617492}, \"4a555da784\": {\"quality\": 0.6274796747967479, \"cost\": 0.06717144000000001, \"time\": 190.02644109725952}, \"4a5cea8b85\": {\"quality\": 0.25497096399535424, \"cost\": 0.08279934000000001, \"time\": 153.70768852233886}, \"4a767339bd\": {\"quality\": 0.6195121951219512, \"cost\": 0.065784288, \"time\": 185.730606508255}, \"4aafd39d76\": {\"quality\": 0.6487804878048781, \"cost\": 0.011711388, \"time\": 195.04439516067504}, \"4aca6e5216\": {\"quality\": 0.4877235772357723, \"cost\": 0.029520160000000007, \"time\": 136.51992263793946}, \"4b18a647d6\": {\"quality\": 0.7812195121951219, \"cost\": 0.09365719600000001, \"time\": 201.93038654327393}, \"4bc4528402\": {\"quality\": 0.6845528455284553, \"cost\": 0.00974208, \"time\": 175.54990649223328}, \"4c158a1a4a\": {\"quality\": 0.47682926829268296, \"cost\": 0.010284996, \"time\": 130.92566409111024}, \"4c954323e3\": {\"quality\": 0.6668757259001161, \"cost\": 0.016642032, \"time\": 211.33878846168517}, \"4d91e8a27b\": {\"quality\": 0.38890824622531933, \"cost\": 0.014601055999999998, \"time\": 149.77316427230835}, \"4dc185389a\": {\"quality\": 0.6353658536585366, \"cost\": 0.063264392, \"time\": 190.5721879005432}, \"4dd3635bc3\": {\"quality\": 0.6648548199767712, \"cost\": 0.026166204, \"time\": 156.0182451725006}, \"4dd98ef398\": {\"quality\": 0.48878048780487804, \"cost\": 0.06819976000000001, \"time\": 104.27807059288025}, \"4dfacd0007\": {\"quality\": 0.8142973286875724, \"cost\": 0.037908216, \"time\": 182.91077904701234}, \"4e298ee0d4\": {\"quality\": 0.5714634146341463, \"cost\": 0.012921647999999999, \"time\": 178.78159699440002}, \"4e3443a0f9\": {\"quality\": 0.6566550522648084, \"cost\": 0.022017672, \"time\": 196.2869851589203}, \"4e4b9db2b8\": {\"quality\": 0.3020325203252033, \"cost\": 0.010630452000000002, \"time\": 134.6365571498871}, \"4e6509f614\": {\"quality\": 0.5430894308943089, \"cost\": 0.0084714, \"time\": 103.83811821937562}, \"4e6a83e751\": {\"quality\": 0.4271544715447154, \"cost\": 0.014789456000000001, \"time\": 1661.4491950511933}, \"4e79c8947f\": {\"quality\": 0.6573170731707317, \"cost\": 0.016321300000000004, \"time\": 72.1683876991272}, \"4e9504432b\": {\"quality\": 0.667479674796748, \"cost\": 0.07073244000000001, \"time\": 194.78891081809996}, \"4e962170dc\": {\"quality\": 0.6195121951219512, \"cost\": 0.06384445600000002, \"time\": 125.06455044746399}, \"4eb0826f21\": {\"quality\": 0.6195121951219512, \"cost\": 0.05706447600000001, \"time\": 84.61785073280335}, \"4ed41bf2e4\": {\"quality\": 0.5291056910569105, \"cost\": 0.079910268, \"time\": 152.21971316337584}, \"4f78672528\": {\"quality\": 0.7267711962833914, \"cost\": 0.073045852, \"time\": 149.74157452583313}, \"4f8cca1195\": {\"quality\": 0.7037514518002322, \"cost\": 0.021906119999999998, \"time\": 222.39478406906127}, \"500860eaa2\": {\"quality\": 0.6195121951219512, \"cost\": 0.14549248799999998, \"time\": 2286.8156877040865}, \"50701b505e\": {\"quality\": 0.6195121951219512, \"cost\": 0.01093236, \"time\": 183.55579237937928}, \"51583a901c\": {\"quality\": 0.5827526132404182, \"cost\": 0.020865352, \"time\": 207.0611351966858}, \"51aeaf9f3e\": {\"quality\": 0.6361788617886179, \"cost\": 0.02071614, \"time\": 60.56362895965576}, \"520b52b64c\": {\"quality\": 0.6005458768873403, \"cost\": 0.07544150000000002, \"time\": 98.03264541625977}, \"521314dab6\": {\"quality\": 0.5414634146341464, \"cost\": 0.011220016000000001, \"time\": 111.50055747032165}, \"5226eb7ff6\": {\"quality\": 0.7410569105691057, \"cost\": 0.028896044, \"time\": 154.04090991020203}, \"526878b5eb\": {\"quality\": 0.47317073170731705, \"cost\": 0.016668683999999996, \"time\": 211.7058870792389}, \"52c1cba6ce\": {\"quality\": 0.6341463414634146, \"cost\": 0.01895658, \"time\": 214.20635576248168}, \"52f041a70e\": {\"quality\": 0.715691056910569, \"cost\": 0.00206075, \"time\": 69.41744899749756}, \"533867574b\": {\"quality\": 0.6173054587688733, \"cost\": 0.06281091200000001, \"time\": 141.25713596343994}, \"53869388bb\": {\"quality\": 0.35284552845528455, \"cost\": 0.011996687999999997, \"time\": 385.71526923179624}, \"53aefd41e4\": {\"quality\": 0.5982229965156796, \"cost\": 0.03475750400000001, \"time\": 124.53529238700867}, \"53d2932c4f\": {\"quality\": 0.5304065040650406, \"cost\": 0.012760976000000002, \"time\": 143.55552654266359}, \"54375d3eba\": {\"quality\": 0.6189430894308943, \"cost\": 0.024152808000000005, \"time\": 112.89141573905945}, \"5474247f91\": {\"quality\": 0.6195121951219512, \"cost\": 0.012371476000000001, \"time\": 125.63727469444275}, \"54993bc472\": {\"quality\": 0.6341463414634146, \"cost\": 0.057053648, \"time\": 83.04090037345887}, \"55358f2285\": {\"quality\": 0.6589430894308943, \"cost\": 0.06846545600000001, \"time\": 137.55791845321656}, \"5569b4f878\": {\"quality\": 0.7922648083623692, \"cost\": 0.021984700000000003, \"time\": 105.12538561820983}, \"557d2cf7ba\": {\"quality\": 0.6195121951219512, \"cost\": 0.012524256000000001, \"time\": 145.9854365348816}, \"55c8aa8935\": {\"quality\": 0.40569105691056906, \"cost\": 0.018154683999999997, \"time\": 185.0385425567627}, \"55e6bf8f14\": {\"quality\": 0.4878048780487805, \"cost\": 0.06334899600000002, \"time\": 175.02917833328246}, \"56a0660622\": {\"quality\": 0.5082926829268293, \"cost\": 0.05803248000000001, \"time\": 128.3964723110199}, \"56a29a28c5\": {\"quality\": 0.6195121951219512, \"cost\": 0.006270012, \"time\": 96.01242618560791}, \"56c4fd5056\": {\"quality\": 0.6195121951219512, \"cost\": 0.128267136, \"time\": 190.17179284095764}, \"5703697dbd\": {\"quality\": 0.13241579558652733, \"cost\": 0.009410826, \"time\": 197.99875745773315}, \"5718f2ed80\": {\"quality\": 0.7897909407665504, \"cost\": 0.0071524019999999995, \"time\": 187.86544318199157}, \"572a02a59a\": {\"quality\": 0.6716260162601626, \"cost\": 0.006410382000000001, \"time\": 171.54228215217591}, \"5750713a41\": {\"quality\": 0.5406504065040652, \"cost\": 0.008179103999999998, \"time\": 157.33118991851808}, \"57757ef15e\": {\"quality\": 0.5353658536585366, \"cost\": 0.011508492, \"time\": 147.7572299003601}, \"579c81bbe0\": {\"quality\": 0.6146341463414634, \"cost\": 0.007194356000000001, \"time\": 109.05034627914429}, \"57bed1722f\": {\"quality\": 0.6195121951219512, \"cost\": 0.00790296, \"time\": 105.67013120651245}, \"585ba6d20b\": {\"quality\": 0.6342973286875726, \"cost\": 0.016752543999999998, \"time\": 243.40559344291688}, \"589267ac64\": {\"quality\": 0.5423112659698026, \"cost\": 0.01900224, \"time\": 108.88611903190613}, \"589a1cea79\": {\"quality\": 0.6195121951219512, \"cost\": 0.014395692000000002, \"time\": 192.94557881355286}, \"58ca42839b\": {\"quality\": 0.7067479674796748, \"cost\": 0.042082800000000004, \"time\": 164.78914923667907}, \"58dc373441\": {\"quality\": 0.48292682926829267, \"cost\": 0.062875472, \"time\": 206.20198302268983}, \"59006532b4\": {\"quality\": 0.4398373983739837, \"cost\": 0.01421172, \"time\": 209.91840863227844}, \"59326c4e00\": {\"quality\": 0.6195121951219512, \"cost\": 0.009057812, \"time\": 105.74859399795533}, \"593975c75b\": {\"quality\": 0.688780487804878, \"cost\": 0.09570490400000001, \"time\": 229.09487490653993}, \"596b4f8694\": {\"quality\": 0.4878048780487805, \"cost\": 0.066254572, \"time\": 204.71900162696838}, \"5971ba4e0d\": {\"quality\": 0.46154471544715453, \"cost\": 0.02107208, \"time\": 666.5095714569092}, \"5996465c0a\": {\"quality\": 0.7142276422764228, \"cost\": 0.0669193, \"time\": 276.37003965377806}, \"59d70b9f65\": {\"quality\": 0.7408943089430894, \"cost\": 0.06718589000000001, \"time\": 181.75046825408936}, \"59f887b67c\": {\"quality\": 0.6195121951219512, \"cost\": 0.015060804, \"time\": 185.60658679008483}, \"5a22920db4\": {\"quality\": 0.7369802555168409, \"cost\": 0.004994448, \"time\": 107.91197681427002}, \"5a35020d45\": {\"quality\": 0.6195121951219512, \"cost\": 0.012432768, \"time\": 174.93387031555176}, \"5aa43da1fc\": {\"quality\": 0.6634959349593497, \"cost\": 0.04908599600000001, \"time\": 142.87173461914062}, \"5ae0d88127\": {\"quality\": 0.6195121951219512, \"cost\": 0.07555794, \"time\": 205.7282470703125}, \"5b10fbdbe1\": {\"quality\": 0.443089430894309, \"cost\": 0.022101736000000004, \"time\": 206.7174467563629}, \"5bade9eb85\": {\"quality\": 0.686225319396051, \"cost\": 0.014263044000000002, \"time\": 240.30284028053285}, \"5be16744bf\": {\"quality\": 0.40822299651567945, \"cost\": 0.020666648000000003, \"time\": 183.62090783119203}, \"5c5055e252\": {\"quality\": 0.6341463414634146, \"cost\": 0.057907604, \"time\": 170.79225449562074}, \"5c53feccd9\": {\"quality\": 0.535191637630662, \"cost\": 0.022622808, \"time\": 214.88010306358336}, \"5c77c7c2b2\": {\"quality\": 0.486829268292683, \"cost\": 0.01199347, \"time\": 164.7836329460144}, \"5d298b5b48\": {\"quality\": 0.5081300813008129, \"cost\": 0.07153200800000001, \"time\": 211.4806851863861}, \"5d41515d2e\": {\"quality\": 0.6988037166085945, \"cost\": 0.041189079999999996, \"time\": 152.06495885849}, \"5d4babc723\": {\"quality\": 0.2809291521486643, \"cost\": 0.06604162400000001, \"time\": 172.66046080589294}, \"5d79b50feb\": {\"quality\": 0.3548780487804878, \"cost\": 0.018818415999999998, \"time\": 109.70083494186402}, \"5dc216cd6b\": {\"quality\": 0.6195121951219512, \"cost\": 0.014456452, \"time\": 143.22672152519226}, \"5dd68c1b8f\": {\"quality\": 0.6670731707317074, \"cost\": 0.002938692, \"time\": 95.08386464118958}, \"5de4a882c1\": {\"quality\": 0.3775609756097561, \"cost\": 0.018079487999999998, \"time\": 163.3632432937622}, \"5e04e1c72d\": {\"quality\": 0.5864227642276423, \"cost\": 0.060815928000000005, \"time\": 139.02201280593872}, \"5e923cee9e\": {\"quality\": 0.7142740998838559, \"cost\": 0.037600364, \"time\": 188.52763104438782}, \"5ea2fab380\": {\"quality\": 0.5060975609756098, \"cost\": 0.006009479999999999, \"time\": 89.62659049034119}, \"5eb3bb525b\": {\"quality\": 0.6526829268292682, \"cost\": 0.012761808000000003, \"time\": 132.20405316352844}, \"5eea899380\": {\"quality\": 0.4015679442508711, \"cost\": 0.07097477600000002, \"time\": 210.64328713417052}, \"5f37b3902b\": {\"quality\": 0.6195121951219512, \"cost\": 0.010051332, \"time\": 109.50351824760438}, \"5f9282df3c\": {\"quality\": 0.37855981416957035, \"cost\": 0.040870088000000006, \"time\": 185.25703506469728}, \"6019884cf3\": {\"quality\": 0.698931475029036, \"cost\": 0.06932, \"time\": 240.45240106582642}, \"606352363e\": {\"quality\": 0.19362369337979093, \"cost\": 0.070268456, \"time\": 234.0612476825714}, \"608728f868\": {\"quality\": 0.6, \"cost\": 0.068669716, \"time\": 218.80884642601012}, \"60b9e936f1\": {\"quality\": 0.6195121951219512, \"cost\": 0.056154416000000006, \"time\": 87.54008474349976}, \"60cb623c53\": {\"quality\": 0.46341463414634154, \"cost\": 0.004160484000000001, \"time\": 67.07470922470092}, \"6234de86b4\": {\"quality\": 0.680650406504065, \"cost\": 0.08330867000000002, \"time\": 161.33356328010558}, \"62352c6854\": {\"quality\": 0.347479674796748, \"cost\": 0.07185688, \"time\": 218.85872540473937}, \"63a0aaebed\": {\"quality\": 0.6195121951219512, \"cost\": 0.005112576000000001, \"time\": 118.10645513534546}, \"647dda686f\": {\"quality\": 0.740650406504065, \"cost\": 0.038015924, \"time\": 197.1805561542511}, \"6511b21ded\": {\"quality\": 0.5281068524970964, \"cost\": 0.036544824, \"time\": 129.3101086616516}, \"652c0f4bdf\": {\"quality\": 0.6195121951219512, \"cost\": 0.01854406, \"time\": 200.11599526405334}, \"6533c85913\": {\"quality\": 0.671869918699187, \"cost\": 0.016586988, \"time\": 216.59043788909912}, \"65627426e0\": {\"quality\": 0.5750058072009292, \"cost\": 0.011773232000000003, \"time\": 1798.7506415367127}, \"65b76da9c6\": {\"quality\": 0.6195121951219512, \"cost\": 0.01047374, \"time\": 144.71517310142517}, \"65be1c1306\": {\"quality\": 0.4783739837398374, \"cost\": 0.044860399999999995, \"time\": 1590.2891113758087}, \"65e0216208\": {\"quality\": 0.6195121951219512, \"cost\": 0.01263888, \"time\": 234.11023540496825}, \"65eee615d7\": {\"quality\": 0.6295934959349594, \"cost\": 0.0011287079999999998, \"time\": 67.5095160961151}, \"6623d7a5ac\": {\"quality\": 0.7706504065040651, \"cost\": 0.067642702, \"time\": 238.91902923583984}, \"66750c0934\": {\"quality\": 0.7448896631823461, \"cost\": 0.013174562, \"time\": 184.41564054489135}, \"66776ec181\": {\"quality\": 0.5121951219512195, \"cost\": 0.014528544, \"time\": 218.20506610870362}, \"66e5ae0a21\": {\"quality\": 0.6195121951219512, \"cost\": 0.0327125, \"time\": 125.57788677215576}, \"6750a8d7a7\": {\"quality\": 0.7970847851335656, \"cost\": 0.03363056, \"time\": 169.0002564907074}, \"67632141f6\": {\"quality\": 0.6195121951219512, \"cost\": 0.007475844000000001, \"time\": 102.3034242630005}, \"67868fcff6\": {\"quality\": 0.44991869918699184, \"cost\": 0.019169336000000002, \"time\": 185.75657877922058}, \"67bab6732d\": {\"quality\": 0.38455284552845526, \"cost\": 0.016392688, \"time\": 159.32311158180238}, \"67fe399cf1\": {\"quality\": 0.624390243902439, \"cost\": 0.010455704, \"time\": 154.35488867759705}, \"6846bd8fb3\": {\"quality\": 0.3334146341463414, \"cost\": 0.009431328, \"time\": 157.63436169624327}, \"68b4cc3e39\": {\"quality\": 0.44522648083623695, \"cost\": 0.09505392000000001, \"time\": 195.20148849487305}, \"69a029ae36\": {\"quality\": 0.6195121951219512, \"cost\": 0.042185764, \"time\": 178.9181830883026}, \"69b3b67de6\": {\"quality\": 0.5991869918699188, \"cost\": 0.016261864, \"time\": 182.98831782341003}, \"69bf3f6ba0\": {\"quality\": 0.5317073170731708, \"cost\": 0.012870332000000002, \"time\": 144.30875706672668}, \"6a022c3f73\": {\"quality\": 0.6390243902439025, \"cost\": 0.006610608, \"time\": 128.81844487190247}, \"6a10c53ad8\": {\"quality\": 0.7610220673635307, \"cost\": 0.018649594, \"time\": 219.1147134780884}, \"6a6348f69d\": {\"quality\": 0.4230894308943089, \"cost\": 0.014753328, \"time\": 1653.7405955314637}, \"6a74a11bee\": {\"quality\": 0.5999303135888503, \"cost\": 0.06506421600000001, \"time\": 158.56066660881044}, \"6a90ec29dd\": {\"quality\": 0.26048780487804873, \"cost\": 0.032533988, \"time\": 121.64445943832398}, \"6aac59742a\": {\"quality\": 0.6195121951219512, \"cost\": 0.015229139999999999, \"time\": 210.34569411277772}, \"6b0862c597\": {\"quality\": 0.6967479674796748, \"cost\": 0.061550596, \"time\": 178.11355109214782}, \"6b3c16def2\": {\"quality\": 0.46341463414634154, \"cost\": 0.002938128, \"time\": 90.25655012130737}, \"6b99b0e901\": {\"quality\": 0.5002439024390244, \"cost\": 0.036148564, \"time\": 119.39121503829956}, \"6b9b2b3515\": {\"quality\": 0.3779326364692218, \"cost\": 0.088494424, \"time\": 215.23190116882324}, \"6bcc02962b\": {\"quality\": 0.6769105691056911, \"cost\": 0.050208152000000006, \"time\": 122.08708581924438}, \"6c1987a9e3\": {\"quality\": 0.5414634146341464, \"cost\": 0.06623874400000002, \"time\": 188.0838879108429}, \"6c50123ee1\": {\"quality\": 0.5729268292682927, \"cost\": 0.05634123, \"time\": 96.05054454803467}, \"6c67c36480\": {\"quality\": 0.6195121951219512, \"cost\": 0.066650968, \"time\": 282.4478175640106}, \"6c9b9f1363\": {\"quality\": 0.5295121951219512, \"cost\": 0.06645499, \"time\": 267.66957621574403}, \"6cc813aa68\": {\"quality\": 0.6195121951219512, \"cost\": 0.008967600000000001, \"time\": 110.58158135414124}, \"6d20c6ace0\": {\"quality\": 0.5440301974448316, \"cost\": 0.043662036, \"time\": 164.7413963794708}, \"6d444fe21a\": {\"quality\": 0.43720092915214864, \"cost\": 0.028377092000000003, \"time\": 181.5676950931549}, \"6db70dc3b6\": {\"quality\": 0.5349593495934959, \"cost\": 0.018471248000000003, \"time\": 1839.0920428276063}, \"6e0690f576\": {\"quality\": 0.6195121951219512, \"cost\": 0.003072384, \"time\": 123.15826468467712}, \"6e06cc804f\": {\"quality\": 0.7730894308943089, \"cost\": 0.122657024, \"time\": 161.19403052330017}, \"6e24048a2e\": {\"quality\": 0.6522764227642277, \"cost\": 0.032542016, \"time\": 137.39676880836487}, \"6e93514f45\": {\"quality\": 0.6195121951219512, \"cost\": 0.008316468, \"time\": 125.78651990890503}, \"6eae47102b\": {\"quality\": 0.4591869918699187, \"cost\": 0.012840492, \"time\": 157.67578744888306}, \"6ed4cae469\": {\"quality\": 0.6234959349593496, \"cost\": 0.021061364000000003, \"time\": 89.84902305603028}, \"6ef3b7127e\": {\"quality\": 0.5697560975609757, \"cost\": 0.012670767999999999, \"time\": 80.09802565574645}, \"6f60a05c33\": {\"quality\": 0.6195121951219512, \"cost\": 0.010392396, \"time\": 162.94261040687562}, \"6fe0b3f929\": {\"quality\": 0.6195121951219512, \"cost\": 0.0075620999999999996, \"time\": 134.45373644828797}, \"700ab1d309\": {\"quality\": 0.6195121951219512, \"cost\": 0.02821224, \"time\": 147.49021158218383}, \"7040e83d52\": {\"quality\": 0.5623693379790942, \"cost\": 0.040416854, \"time\": 170.36972126960754}, \"7046765af8\": {\"quality\": 0.200801393728223, \"cost\": 0.06678788599999999, \"time\": 205.17822675704957}, \"70b7c92ce8\": {\"quality\": 0.6439024390243903, \"cost\": 0.06304620400000002, \"time\": 216.9712851047516}, \"70c850e039\": {\"quality\": 0.6195121951219512, \"cost\": 0.00682044, \"time\": 111.72890758514404}, \"7112a7e64c\": {\"quality\": 0.5082113821138211, \"cost\": 0.008551890000000001, \"time\": 102.93057270050049}, \"71b615468b\": {\"quality\": 0.31138211382113823, \"cost\": 0.016398592, \"time\": 133.36204581260682}, \"723fd5589a\": {\"quality\": 0.6195121951219512, \"cost\": 0.025608394, \"time\": 148.1404324531555}, \"7250da0f41\": {\"quality\": 0.6195121951219512, \"cost\": 0.010252872, \"time\": 102.41908440589904}, \"7274a50778\": {\"quality\": 0.5536585365853659, \"cost\": 0.06315958, \"time\": 127.02501587867737}, \"72d022ce33\": {\"quality\": 0.8057259001161441, \"cost\": 0.08692716000000002, \"time\": 196.42877383232116}, \"7347cf0308\": {\"quality\": 0.6195121951219512, \"cost\": 0.007518113999999999, \"time\": 112.80636467933655}, \"736e652158\": {\"quality\": 0.6195121951219512, \"cost\": 0.010933332, \"time\": 202.134419298172}, \"739b1f81dc\": {\"quality\": 0.7352845528455284, \"cost\": 0.005161632, \"time\": 102.21428322792053}, \"742a5c0552\": {\"quality\": 0.6585365853658537, \"cost\": 0.05980691200000001, \"time\": 95.91068015098571}, \"742ec1b2e1\": {\"quality\": 0.691869918699187, \"cost\": 0.06350464000000001, \"time\": 189.14654784202577}, \"7435fd54f8\": {\"quality\": 0.6567131242741, \"cost\": 0.06201505600000001, \"time\": 203.98798232078553}, \"7445d99939\": {\"quality\": 0.4034843205574912, \"cost\": 0.011135484, \"time\": 153.3177831172943}, \"7466a5f424\": {\"quality\": 0.4263414634146342, \"cost\": 0.0244955, \"time\": 218.84586930274963}, \"74a0be215b\": {\"quality\": 0.48292682926829267, \"cost\": 0.06485306800000001, \"time\": 214.73199005126952}, \"74d7f64b8c\": {\"quality\": 0.6195121951219512, \"cost\": 0.011531088000000002, \"time\": 211.1984474658966}, \"7524905580\": {\"quality\": 0.7694889663182346, \"cost\": 0.012599868, \"time\": 157.22432436943055}, \"752d9649f2\": {\"quality\": 0.5467479674796748, \"cost\": 0.022367612000000002, \"time\": 204.80810337066652}, \"75d61c2cd0\": {\"quality\": 0.7049128919860628, \"cost\": 0.062316, \"time\": 160.31273555755615}, \"7604c0aa13\": {\"quality\": 0.6195121951219512, \"cost\": 0.0142076, \"time\": 142.32078042030335}, \"76c09db721\": {\"quality\": 0.7820441347270615, \"cost\": 0.04096148000000001, \"time\": 236.8196849822998}, \"774f268b66\": {\"quality\": 0.6810569105691057, \"cost\": 0.08915512, \"time\": 201.10393743515016}, \"7765576286\": {\"quality\": 0.567479674796748, \"cost\": 0.016253124, \"time\": 222.17675213813783}, \"77c02b00c1\": {\"quality\": 0.5561091753774681, \"cost\": 0.017938308, \"time\": 201.10154690742493}, \"7801da66b9\": {\"quality\": 0.6195121951219512, \"cost\": 0.00746667, \"time\": 114.64372763633727}, \"782d52674e\": {\"quality\": 0.3201974448315911, \"cost\": 0.06483101399999999, \"time\": 193.24506635665892}, \"786e5d0af5\": {\"quality\": 0.6195121951219512, \"cost\": 0.016454096, \"time\": 197.87756876945497}, \"7878563d63\": {\"quality\": 0.1632171893147503, \"cost\": 0.028198652, \"time\": 84.60407543182373}, \"79e1ca9b3c\": {\"quality\": 0.6195121951219512, \"cost\": 0.06867538, \"time\": 110.48155570030212}, \"79fad58f07\": {\"quality\": 0.46674796747967484, \"cost\": 0.029654968000000004, \"time\": 679.6173066616059}, \"7a207b42a8\": {\"quality\": 0.6195121951219512, \"cost\": 0.012325835999999998, \"time\": 163.009268951416}, \"7a2cdc546c\": {\"quality\": 0.2631707317073171, \"cost\": 0.06310833, \"time\": 160.65065565109253}, \"7a42a77788\": {\"quality\": 0.6195121951219512, \"cost\": 0.06019363200000001, \"time\": 138.42030897140503}, \"7a58d3472b\": {\"quality\": 0.5959349593495935, \"cost\": 0.025425438, \"time\": 147.28676762580872}, \"7b3937c1f1\": {\"quality\": 0.39544715447154466, \"cost\": 0.02868438, \"time\": 89.45209522247315}, \"7b6f44618e\": {\"quality\": 0.5699186991869919, \"cost\": 0.018827976000000003, \"time\": 179.5670045375824}, \"7c62576527\": {\"quality\": 0.5152845528455284, \"cost\": 0.042298876, \"time\": 167.9848653316498}, \"7c89a2b69e\": {\"quality\": 0.6195121951219512, \"cost\": 0.019231344, \"time\": 252.72414374351501}, \"7c96c9712f\": {\"quality\": 0.6536585365853659, \"cost\": 0.06122004000000001, \"time\": 127.95261840820312}, \"7ca066aa1c\": {\"quality\": 0.7818002322880371, \"cost\": 0.040026332000000005, \"time\": 187.7031367778778}, \"7cb5591f27\": {\"quality\": 0.3416840882694541, \"cost\": 0.0826827, \"time\": 138.52483682632447}, \"7cf56a7fbc\": {\"quality\": 0.6195121951219512, \"cost\": 0.08315224400000001, \"time\": 106.8193588256836}, \"7d44f0959d\": {\"quality\": 0.46680603948896626, \"cost\": 0.011511138, \"time\": 133.2327687740326}, \"7d60c38c5c\": {\"quality\": 0.7220441347270616, \"cost\": 0.00538134, \"time\": 131.21526570320128}, \"7d9b4535ac\": {\"quality\": 0.6769802555168408, \"cost\": 0.063389518, \"time\": 211.52207164764405}, \"7daf7ff182\": {\"quality\": 0.5284552845528456, \"cost\": 0.017232744, \"time\": 170.8186996459961}, \"7dcedb3d02\": {\"quality\": 0.6593960511033682, \"cost\": 0.06829533600000001, \"time\": 231.27069878578186}, \"7e22f12cd1\": {\"quality\": 0.5735540069686411, \"cost\": 0.03302558, \"time\": 186.45001711845399}, \"7e53a50b13\": {\"quality\": 0.4565737514518003, \"cost\": 0.033876876, \"time\": 151.98724694252013}, \"7ed07ad40a\": {\"quality\": 0.6177700348432056, \"cost\": 0.0116127, \"time\": 211.14939546585083}, \"7fa67a7656\": {\"quality\": 0.5077235772357723, \"cost\": 0.01125696, \"time\": 107.8444951057434}, \"7fc6c84bdf\": {\"quality\": 0.7865969802555168, \"cost\": 0.03407424, \"time\": 176.73210568428038}, \"7ff8a779cc\": {\"quality\": 0.7043089430894309, \"cost\": 0.06711379800000002, \"time\": 168.2764458656311}, \"801af99400\": {\"quality\": 0.7181765389082462, \"cost\": 0.07170897000000001, \"time\": 184.03885922431945}, \"806881adcb\": {\"quality\": 0.6209175377468059, \"cost\": 0.012441478, \"time\": 130.4481972694397}, \"80ad4122e4\": {\"quality\": 0.6211382113821139, \"cost\": 0.07175158800000002, \"time\": 199.85183753967286}, \"80be7df955\": {\"quality\": 0.6195121951219512, \"cost\": 0.0029887200000000003, \"time\": 99.58167638778687}, \"80bf60c422\": {\"quality\": 0.6195121951219512, \"cost\": 0.005357952000000001, \"time\": 138.18060364723206}, \"81333c7a33\": {\"quality\": 0.5926829268292683, \"cost\": 0.027174752000000003, \"time\": 143.90170640945433}, \"813e75210b\": {\"quality\": 0.3377816492450639, \"cost\": 0.05362793600000001, \"time\": 1273.6602407932282}, \"81660ae8b2\": {\"quality\": 0.6195121951219512, \"cost\": 0.06746300400000001, \"time\": 194.1754295349121}, \"816958b5d1\": {\"quality\": 0.6963414634146342, \"cost\": 0.07060810400000002, \"time\": 230.40107922554017}, \"81ab2ef3f4\": {\"quality\": 0.3821138211382114, \"cost\": 0.030218416000000005, \"time\": 135.14831619262696}, \"829df73946\": {\"quality\": 0.6964227642276423, \"cost\": 0.008790552, \"time\": 111.97322883605958}, \"82ea1bd1b9\": {\"quality\": 0.8085365853658537, \"cost\": 0.069294098, \"time\": 143.64819779396058}, \"8357183895\": {\"quality\": 0.6195121951219512, \"cost\": 0.014406096000000002, \"time\": 247.72333025932312}, \"8392a6083a\": {\"quality\": 0.6195121951219512, \"cost\": 0.017490875999999995, \"time\": 201.15325956344606}, \"83aee532b7\": {\"quality\": 0.6195121951219512, \"cost\": 0.064240072, \"time\": 249.5134038925171}, \"83b26646c3\": {\"quality\": 0.6652845528455285, \"cost\": 0.04101521600000001, \"time\": 139.2657244682312}, \"83c9e66ec6\": {\"quality\": 0.6439024390243903, \"cost\": 0.012396760000000001, \"time\": 179.99957365989684}, \"847a0e5db5\": {\"quality\": 0.6909756097560976, \"cost\": 0.06664256800000001, \"time\": 183.08391642570496}, \"847fd49235\": {\"quality\": 0.3338327526132404, \"cost\": 0.006645311999999999, \"time\": 167.98347339630126}, \"849100224d\": {\"quality\": 0.3345528455284553, \"cost\": 0.08884810000000001, \"time\": 220.43858489990234}, \"84b91c37ab\": {\"quality\": 0.24628339140534267, \"cost\": 0.069210264, \"time\": 236.29133720397948}, \"8519bef585\": {\"quality\": 0.6195121951219512, \"cost\": 0.005065452, \"time\": 122.15238361358642}, \"85c94a5505\": {\"quality\": 0.4611382113821138, \"cost\": 0.056482279999999996, \"time\": 1274.952497625351}, \"85eda38404\": {\"quality\": 0.6357142857142857, \"cost\": 0.01686636, \"time\": 103.26268811225891}, \"862183bfb9\": {\"quality\": 0.6951219512195121, \"cost\": 0.012958452, \"time\": 162.91549973487855}, \"8631e49c94\": {\"quality\": 0.6439024390243903, \"cost\": 0.06574006400000001, \"time\": 227.21288151741027}, \"8668f65f05\": {\"quality\": 0.6195121951219512, \"cost\": 0.01596044, \"time\": 191.07359557151796}, \"86bf6375af\": {\"quality\": 0.6560975609756097, \"cost\": 0.06964888, \"time\": 260.4157979488373}, \"870e2f87b4\": {\"quality\": 0.7660046457607433, \"cost\": 0.025822204000000005, \"time\": 201.3596661567688}, \"887ad124e1\": {\"quality\": 0.709349593495935, \"cost\": 0.0070279439999999995, \"time\": 150.77690081596376}, \"8886cb3082\": {\"quality\": 0.6834843205574914, \"cost\": 0.019317688000000003, \"time\": 211.99706134796142}, \"88e71efa9b\": {\"quality\": 0.28095238095238095, \"cost\": 0.046789276000000005, \"time\": 186.75326628684996}, \"8941621423\": {\"quality\": 0.6786991869918699, \"cost\": 0.012530784, \"time\": 167.57245116233827}, \"8950a6efe0\": {\"quality\": 0.48292682926829267, \"cost\": 0.05844699200000001, \"time\": 136.73901495933532}, \"8961e4d901\": {\"quality\": 0.6195121951219512, \"cost\": 0.010754568, \"time\": 156.90380158424378}, \"8974aa89a0\": {\"quality\": 0.5727642276422764, \"cost\": 0.01088668, \"time\": 377.91359758377075}, \"89836d2020\": {\"quality\": 0.6195121951219512, \"cost\": 0.06319070400000001, \"time\": 132.40995478630066}, \"89a289907e\": {\"quality\": 0.7753426248548199, \"cost\": 0.014910682, \"time\": 212.81768522262573}, \"89a35a09b1\": {\"quality\": 0.525609756097561, \"cost\": 0.011930976, \"time\": 194.43942880630493}, \"8a3a35c762\": {\"quality\": 0.6390243902439025, \"cost\": 0.07371796400000001, \"time\": 188.05857005119324}, \"8a50695d1f\": {\"quality\": 0.5045644599303136, \"cost\": 0.027424088000000003, \"time\": 111.09706916809083}, \"8ab351aa13\": {\"quality\": 0.6526829268292682, \"cost\": 0.089840192, \"time\": 137.60196342468262}, \"8ac8b5773a\": {\"quality\": 0.6292682926829268, \"cost\": 0.06544508800000001, \"time\": 216.46768646240236}, \"8acd758b7f\": {\"quality\": 0.6600929152148665, \"cost\": 0.008952419999999999, \"time\": 151.0333396911621}, \"8b10891ea5\": {\"quality\": 0.4276422764227642, \"cost\": 0.07350195200000001, \"time\": 188.96431422233582}, \"8b721bbc6f\": {\"quality\": 0.6195121951219512, \"cost\": 0.012898752, \"time\": 215.82190022468566}, \"8b77535cce\": {\"quality\": 0.7979558652729384, \"cost\": 0.07287564, \"time\": 215.97581152915956}, \"8bbbe0f52a\": {\"quality\": 0.6195121951219512, \"cost\": 0.008104248, \"time\": 149.25933980941772}, \"8bc184f385\": {\"quality\": 0.37040650406504066, \"cost\": 0.008849639999999999, \"time\": 130.85588617324828}, \"8bf5c3eadc\": {\"quality\": 0.6195121951219512, \"cost\": 0.008037444, \"time\": 130.57925519943237}, \"8bf80a50cb\": {\"quality\": 0.45910569105691057, \"cost\": 0.06549705000000001, \"time\": 191.59308562278747}, \"8c274ca255\": {\"quality\": 0.7277932636469222, \"cost\": 0.07598581600000001, \"time\": 203.60676689147948}, \"8d7594020b\": {\"quality\": 0.6195121951219512, \"cost\": 0.06030525600000001, \"time\": 142.03003664016722}, \"8d79e03266\": {\"quality\": 0.728513356562137, \"cost\": 0.019196192, \"time\": 166.50991864204406}, \"8d90814b94\": {\"quality\": 0.35382113821138217, \"cost\": 0.019129876, \"time\": 218.55778388977052}, \"8e1a01da19\": {\"quality\": 0.6195121951219512, \"cost\": 0.06321022000000001, \"time\": 206.71381578445434}, \"8e2498635d\": {\"quality\": 0.8135772357723576, \"cost\": 0.142084164, \"time\": 153.71730332374574}, \"8e5842ccbd\": {\"quality\": 0.7329268292682927, \"cost\": 0.007325802000000001, \"time\": 112.95453724861144}, \"8e5daf241e\": {\"quality\": 0.6923228803716608, \"cost\": 0.060446860000000005, \"time\": 186.30223965644836}, \"8e9715ee01\": {\"quality\": 0.5639837398373984, \"cost\": 0.08655280400000002, \"time\": 139.08190727233887}, \"8e9b7300d4\": {\"quality\": 0.4639837398373984, \"cost\": 0.06058211200000001, \"time\": 150.15705633163452}, \"8f29fab8ac\": {\"quality\": 0.4739140534262486, \"cost\": 0.06764498000000001, \"time\": 200.2176959514618}, \"8f44d89429\": {\"quality\": 0.7065040650406504, \"cost\": 0.069996624, \"time\": 185.83366765975953}, \"8f4caddfe6\": {\"quality\": 0.5421138211382114, \"cost\": 0.021058936, \"time\": 161.7523732662201}, \"8f4edde3f0\": {\"quality\": 0.5726480836236936, \"cost\": 0.03185762, \"time\": 159.63181648254394}, \"8f9cefbc22\": {\"quality\": 0.6585365853658537, \"cost\": 0.05867756800000001, \"time\": 105.53310813903809}, \"9025e2480f\": {\"quality\": 0.6595934959349593, \"cost\": 0.00965967, \"time\": 128.92424964904785}, \"9028588af4\": {\"quality\": 0.35292682926829266, \"cost\": 0.09341379200000001, \"time\": 1090.4150963783263}, \"9059fd80ad\": {\"quality\": 0.5902439024390244, \"cost\": 0.027686471999999997, \"time\": 2047.5434857845307}, \"90d5e40c1b\": {\"quality\": 0.6195121951219512, \"cost\": 0.007589304, \"time\": 120.88176670074463}, \"90d9a86a2a\": {\"quality\": 0.6195121951219512, \"cost\": 0.0036013499999999997, \"time\": 98.5603612422943}, \"90ff13783c\": {\"quality\": 0.6255284552845529, \"cost\": 0.018824976, \"time\": 219.693346452713}, \"90ff8eb055\": {\"quality\": 0.5428571428571429, \"cost\": 0.015549384, \"time\": 206.51184477806092}, \"9104e31369\": {\"quality\": 0.6640650406504065, \"cost\": 0.022935780000000003, \"time\": 229.26985802650452}, \"918983323f\": {\"quality\": 0.5085365853658537, \"cost\": 0.045187920000000006, \"time\": 1981.811208820343}, \"91928dfdd9\": {\"quality\": 0.6747967479674797, \"cost\": 0.06523348, \"time\": 182.1119598388672}, \"91c800af6b\": {\"quality\": 0.6161904761904762, \"cost\": 0.018671568000000003, \"time\": 212.29187331199645}, \"91e841cfd5\": {\"quality\": 0.6195121951219512, \"cost\": 0.067784204, \"time\": 200.92469301223755}, \"9253901a1f\": {\"quality\": 0.6195121951219512, \"cost\": 0.07052953600000002, \"time\": 216.47048025131227}, \"9288642e53\": {\"quality\": 0.6195121951219512, \"cost\": 0.008932916, \"time\": 94.32965250015259}, \"92ba9c5be3\": {\"quality\": 0.567479674796748, \"cost\": 0.064902632, \"time\": 138.75575132369994}, \"92c9dcd43b\": {\"quality\": 0.6195121951219512, \"cost\": 0.014990692, \"time\": 237.96368551254272}, \"93011c0821\": {\"quality\": 0.6195121951219512, \"cost\": 0.019036675999999995, \"time\": 267.35766806602476}, \"933b4d17dd\": {\"quality\": 0.5723577235772358, \"cost\": 0.012143372, \"time\": 172.0639699459076}, \"9373267bdb\": {\"quality\": 0.5410452961672474, \"cost\": 0.06831846000000001, \"time\": 258.8440625667572}, \"94010928c6\": {\"quality\": 0.25325203252032524, \"cost\": 0.011834136, \"time\": 131.6460223197937}, \"9403809e44\": {\"quality\": 0.4588617886178861, \"cost\": 0.016092682, \"time\": 231.8062086582184}, \"940c88ddc5\": {\"quality\": 0.5727874564459932, \"cost\": 0.029609188, \"time\": 126.90794463157654}, \"948f4081ba\": {\"quality\": 0.6792102206736355, \"cost\": 0.039854912000000006, \"time\": 248.46457772254945}, \"94ac356663\": {\"quality\": 0.5626016260162602, \"cost\": 0.012098179999999998, \"time\": 162.48227286338806}, \"94dff9a424\": {\"quality\": 0.7495934959349594, \"cost\": 0.013244731999999999, \"time\": 187.24048733711243}, \"9508356a2e\": {\"quality\": 0.6195121951219512, \"cost\": 0.012070511999999999, \"time\": 184.9062297821045}, \"9539d0e28c\": {\"quality\": 0.37120789779326363, \"cost\": 0.04079825200000001, \"time\": 198.3246217250824}, \"956bdcc254\": {\"quality\": 0.12377468060394889, \"cost\": 0.026780572000000002, \"time\": 157.82376141548156}, \"957d0dafc5\": {\"quality\": 0.5816260162601626, \"cost\": 0.067579808, \"time\": 153.21978812217714}, \"9594b0c783\": {\"quality\": 0.6195121951219512, \"cost\": 0.006768972, \"time\": 171.20207090377806}, \"95a7b80c2a\": {\"quality\": 0.797979094076655, \"cost\": 0.02950122, \"time\": 181.7308371067047}, \"964c671f18\": {\"quality\": 0.7772822299651567, \"cost\": 0.021085212, \"time\": 212.14437518119811}, \"9679fe2b69\": {\"quality\": 0.6195121951219512, \"cost\": 0.007065155999999999, \"time\": 184.30773782730103}, \"968fc95038\": {\"quality\": 0.4525319396051103, \"cost\": 0.005385312, \"time\": 147.3559940338135}, \"96b487c724\": {\"quality\": 0.440650406504065, \"cost\": 0.0038495599999999993, \"time\": 780.6981408596039}, \"96c30205f5\": {\"quality\": 0.5457723577235772, \"cost\": 0.064298388, \"time\": 146.8725693702698}, \"96f87d6483\": {\"quality\": 0.7184436701509872, \"cost\": 0.021446996000000003, \"time\": 236.88481707572936}, \"972c83b002\": {\"quality\": 0.6195121951219512, \"cost\": 0.008979192, \"time\": 162.82133150100708}, \"977a4d6b6b\": {\"quality\": 0.28170731707317076, \"cost\": 0.021416136000000002, \"time\": 220.9775969028473}, \"97bc30bd83\": {\"quality\": 0.21211382113821137, \"cost\": 0.015064284, \"time\": 209.8210876464844}, \"980db5f95f\": {\"quality\": 0.6791869918699187, \"cost\": 0.059587308000000005, \"time\": 168.43710894584655}, \"9836765d41\": {\"quality\": 0.6195121951219512, \"cost\": 0.06211643600000001, \"time\": 171.59594984054564}, \"99569e3937\": {\"quality\": 0.6777932636469222, \"cost\": 0.07203462400000002, \"time\": 259.5382764339447}, \"99ea16a9a6\": {\"quality\": 0.815528455284553, \"cost\": 0.034362648, \"time\": 175.2437418460846}, \"9a5b39370f\": {\"quality\": 0.6195121951219512, \"cost\": 0.068652448, \"time\": 237.78598909378053}, \"9aa4abfb50\": {\"quality\": 0.6195121951219512, \"cost\": 0.005156892000000001, \"time\": 114.41352844238281}, \"9ad7a98c31\": {\"quality\": 0.7987108013937282, \"cost\": 0.07103688, \"time\": 182.73348231315612}, \"9b3fb79bcb\": {\"quality\": 0.7849477351916376, \"cost\": 0.028800140000000002, \"time\": 167.58384928703308}, \"9b6d4915f3\": {\"quality\": 0.6195121951219512, \"cost\": 0.0058737600000000004, \"time\": 96.19086871147155}, \"9bae5bafc1\": {\"quality\": 0.4828222996515679, \"cost\": 0.020766276000000004, \"time\": 144.2882854938507}, \"9be8a5f317\": {\"quality\": 0.6097560975609756, \"cost\": 0.070895624, \"time\": 241.7383065700531}, \"9c549db0a7\": {\"quality\": 0.6195121951219512, \"cost\": 0.021638197999999997, \"time\": 138.09213070869447}, \"9c85f8cfcb\": {\"quality\": 0.6195121951219512, \"cost\": 0.004881552, \"time\": 85.23935956954956}, \"9c8cc46e6c\": {\"quality\": 0.6195121951219512, \"cost\": 0.004920108, \"time\": 113.78448638916015}, \"9c97d35a30\": {\"quality\": 0.8032520325203251, \"cost\": 0.011073209999999998, \"time\": 181.72330527305604}, \"9cbe7858a2\": {\"quality\": 0.6195121951219512, \"cost\": 0.06554662800000002, \"time\": 227.9875654697418}, \"9ce2c3fd98\": {\"quality\": 0.6195121951219512, \"cost\": 0.006856308, \"time\": 155.26185359954835}, \"9d18cd0737\": {\"quality\": 0.4644018583042973, \"cost\": 0.006854579999999999, \"time\": 124.20924863815307}, \"9e06360bc9\": {\"quality\": 0.46796747967479674, \"cost\": 0.011727056, \"time\": 160.3625663280487}, \"9f07a95e69\": {\"quality\": 0.5894308943089431, \"cost\": 0.073255632, \"time\": 234.6284239768982}, \"9fb157be35\": {\"quality\": 0.6195121951219512, \"cost\": 0.012696056, \"time\": 159.5446131706238}, \"a04ac8e33a\": {\"quality\": 0.6195121951219512, \"cost\": 0.06885804400000001, \"time\": 157.56720504760742}, \"a04bc6e116\": {\"quality\": 0.5231707317073171, \"cost\": 0.06362432400000001, \"time\": 166.92331409454346}, \"a0b81be5b4\": {\"quality\": 0.3871660859465737, \"cost\": 0.00347016, \"time\": 123.85612168312073}, \"a0c85d260e\": {\"quality\": 0.69602787456446, \"cost\": 0.024224592, \"time\": 235.78984532356262}, \"a0dc9f50ac\": {\"quality\": 0.18065040650406505, \"cost\": 0.007600488, \"time\": 130.9164544582367}, \"a18225b7b5\": {\"quality\": 0.6352032520325204, \"cost\": 0.050534568, \"time\": 152.35800580978395}, \"a1d822289e\": {\"quality\": 0.4399303135888502, \"cost\": 0.08478274000000001, \"time\": 163.93071274757386}, \"a25596c056\": {\"quality\": 0.6195121951219512, \"cost\": 0.067823316, \"time\": 216.80869884490966}, \"a2811c7324\": {\"quality\": 0.6701277584204414, \"cost\": 0.028778920000000003, \"time\": 168.49099044799806}, \"a2aa082d14\": {\"quality\": 0.6195121951219512, \"cost\": 0.022896492, \"time\": 119.0820493221283}, \"a2cd339ad9\": {\"quality\": 0.6406504065040651, \"cost\": 0.018992632, \"time\": 243.06863555908203}, \"a2fd03e6a5\": {\"quality\": 0.47276422764227644, \"cost\": 0.07033531999999999, \"time\": 1132.2907946109772}, \"a31e87d7cb\": {\"quality\": 0.6329268292682927, \"cost\": 0.00273726, \"time\": 105.99368834495544}, \"a3e23c327b\": {\"quality\": 0.5317073170731708, \"cost\": 0.068050688, \"time\": 204.56698241233826}, \"a457f6c300\": {\"quality\": 0.6195121951219512, \"cost\": 0.012840227999999999, \"time\": 168.6292993545532}, \"a47de025c8\": {\"quality\": 0.6195121951219512, \"cost\": 0.007961357999999998, \"time\": 138.18772959709167}, \"a515a9c8cc\": {\"quality\": 0.6253658536585366, \"cost\": 0.005416404, \"time\": 126.60180377960205}, \"a5949b76ec\": {\"quality\": 0.6195121951219512, \"cost\": 0.003754764, \"time\": 81.27935070991516}, \"a60dd076b8\": {\"quality\": 0.405609756097561, \"cost\": 0.0034407720000000004, \"time\": 118.2785505771637}, \"a6297a6c56\": {\"quality\": 0.6195121951219512, \"cost\": 0.005110776, \"time\": 118.47536420822144}, \"a62b7555b9\": {\"quality\": 0.4455284552845528, \"cost\": 0.07002202800000001, \"time\": 229.59565677642823}, \"a63f48e8ca\": {\"quality\": 0.6195121951219512, \"cost\": 0.069574668, \"time\": 132.71091380119324}, \"a6460dbb7c\": {\"quality\": 0.7212078977932637, \"cost\": 0.009528016, \"time\": 130.55586276054382}, \"a66a4cf4b0\": {\"quality\": 0.567479674796748, \"cost\": 0.07641536000000002, \"time\": 202.29455227851867}, \"a6e2d69222\": {\"quality\": 0.4239024390243902, \"cost\": 0.045957064, \"time\": 178.1903151988983}, \"a717c4c535\": {\"quality\": 0.6195121951219512, \"cost\": 0.019736044, \"time\": 234.4358515739441}, \"a76afe9960\": {\"quality\": 0.5696051103368177, \"cost\": 0.060384544000000005, \"time\": 172.19008646011352}, \"a7a6353090\": {\"quality\": 0.4487224157955866, \"cost\": 0.066870862, \"time\": 166.1868350982666}, \"a80f6535b1\": {\"quality\": 0.5308130081300814, \"cost\": 0.020093776, \"time\": 97.89270992279053}, \"a86b137d7f\": {\"quality\": 0.6815447154471544, \"cost\": 0.006451132, \"time\": 93.94514923095703}, \"a88eb1493c\": {\"quality\": 0.41745644599303144, \"cost\": 0.0066783, \"time\": 116.80834813117981}, \"a89c533d6c\": {\"quality\": 0.6195121951219512, \"cost\": 0.06598649999999999, \"time\": 83.00586166381837}, \"a8d8264600\": {\"quality\": 0.4808943089430895, \"cost\": 0.071912272, \"time\": 251.74223246574402}, \"a8eb36b210\": {\"quality\": 0.4832520325203252, \"cost\": 0.03806308, \"time\": 194.64342403411865}, \"a95b4a6dd0\": {\"quality\": 0.6195121951219512, \"cost\": 0.012172187999999999, \"time\": 172.00175199508666}, \"a9621ea4e6\": {\"quality\": 0.47317073170731705, \"cost\": 0.010803696, \"time\": 194.9391739845276}, \"a9721a0a50\": {\"quality\": 0.6624390243902439, \"cost\": 0.009840604, \"time\": 122.32440161705017}, \"a972b02c61\": {\"quality\": 0.7189430894308944, \"cost\": 0.03160314, \"time\": 67.33741846084595}, \"a9c5c4e311\": {\"quality\": 0.7951335656213705, \"cost\": 0.028237476, \"time\": 182.70986766815184}, \"a9d96670eb\": {\"quality\": 0.6195121951219512, \"cost\": 0.05767817200000001, \"time\": 169.8078365802765}, \"a9e8c974d3\": {\"quality\": 0.6195121951219512, \"cost\": 0.06961316799999999, \"time\": 225.65897822380066}, \"aa08180e36\": {\"quality\": 0.6146341463414634, \"cost\": 0.014778952000000001, \"time\": 221.92620844841002}, \"aa38702a02\": {\"quality\": 0.6146341463414634, \"cost\": 0.006915852, \"time\": 164.99627866744996}, \"aadbfc418b\": {\"quality\": 0.45479674796747965, \"cost\": 0.00576612, \"time\": 161.90984616279601}, \"aaeb8b0010\": {\"quality\": 0.6195121951219512, \"cost\": 0.06474566000000001, \"time\": 209.02622961997986}, \"ab288ee7f2\": {\"quality\": 0.5373983739837398, \"cost\": 0.06573288000000001, \"time\": 226.1878888130188}, \"ab43b02cb0\": {\"quality\": 0.6195121951219512, \"cost\": 0.01377632, \"time\": 126.55581045150757}, \"aba1d612cc\": {\"quality\": 0.5083391405342625, \"cost\": 0.09728845600000001, \"time\": 219.89660325050355}, \"ac208e7a1d\": {\"quality\": 0.4333333333333334, \"cost\": 0.01616966, \"time\": 142.93958106040955}, \"ac2224adbe\": {\"quality\": 0.6195121951219512, \"cost\": 0.039715708, \"time\": 125.78687748908996}, \"ac828ffe70\": {\"quality\": 0.5008130081300812, \"cost\": 0.003011208, \"time\": 454.9158252716064}, \"ac9fdc1550\": {\"quality\": 0.5360162601626015, \"cost\": 0.026754198, \"time\": 173.65110726356505}, \"aca957ecff\": {\"quality\": 0.8072590011614402, \"cost\": 0.08830083200000001, \"time\": 230.63441123962403}, \"acfe1ed920\": {\"quality\": 0.35609756097560974, \"cost\": 0.02273798, \"time\": 88.96396307945251}, \"ad3efe44c3\": {\"quality\": 0.6195121951219512, \"cost\": 0.004642056, \"time\": 92.58378911018372}, \"ad41c95a99\": {\"quality\": 0.3482113821138212, \"cost\": 0.045380508, \"time\": 154.1578179359436}, \"ad48432c22\": {\"quality\": 0.6569105691056911, \"cost\": 0.015225708, \"time\": 247.30540533065795}, \"ad6ebbba8d\": {\"quality\": 0.6195121951219512, \"cost\": 0.012972292, \"time\": 209.87320923805237}, \"ad90055ef6\": {\"quality\": 0.6195121951219512, \"cost\": 0.00457182, \"time\": 93.97476525306702}, \"adab1e0fb1\": {\"quality\": 0.46829268292682935, \"cost\": 0.008422604000000002, \"time\": 115.4403871536255}, \"ae655ec593\": {\"quality\": 0.35599303135888505, \"cost\": 0.006649944, \"time\": 167.62679462432862}, \"ae94b172be\": {\"quality\": 0.5552613240418119, \"cost\": 0.013280692, \"time\": 162.66643962860107}, \"aec9dc5873\": {\"quality\": 0.6195121951219512, \"cost\": 0.055165792000000005, \"time\": 132.68569073677062}, \"af360c323c\": {\"quality\": 0.6195121951219512, \"cost\": 0.006864167999999999, \"time\": 195.27329888343812}, \"af90567194\": {\"quality\": 0.7732404181184669, \"cost\": 0.02378152, \"time\": 264.66726565361023}, \"afe77d0f89\": {\"quality\": 0.768513356562137, \"cost\": 0.069933622, \"time\": 251.59823207855226}, \"b0948c05b6\": {\"quality\": 0.6380487804878049, \"cost\": 0.017833904, \"time\": 149.21853432655334}, \"b0c4a6640b\": {\"quality\": 0.4317653890824623, \"cost\": 0.063453446, \"time\": 221.18421225547792}, \"b12caafd58\": {\"quality\": 0.46829268292682935, \"cost\": 0.06446362400000001, \"time\": 217.29487829208375}, \"b18168b9c1\": {\"quality\": 0.3608130081300814, \"cost\": 0.010323167999999999, \"time\": 189.31807408332824}, \"b1a7428a01\": {\"quality\": 0.21260162601626015, \"cost\": 0.06952381600000002, \"time\": 206.16540212631224}, \"b1acdebb48\": {\"quality\": 0.6821138211382114, \"cost\": 0.102856868, \"time\": 183.48106842041017}, \"b1b06f4ee7\": {\"quality\": 0.6195121951219512, \"cost\": 0.12662600000000002, \"time\": 142.81967635154723}, \"b1cf8d33e5\": {\"quality\": 0.3186178861788618, \"cost\": 0.011143266, \"time\": 175.05015845298766}, \"b1e9ab6b1a\": {\"quality\": 0.5263414634146341, \"cost\": 0.07167981200000001, \"time\": 206.59926490783693}, \"b2b057ba41\": {\"quality\": 0.4799186991869918, \"cost\": 0.0052038779999999995, \"time\": 142.83305773735046}, \"b2e063499d\": {\"quality\": 0.6911498257839722, \"cost\": 0.018540096, \"time\": 176.64938292503356}, \"b33412410e\": {\"quality\": 0.6195121951219512, \"cost\": 0.03299608, \"time\": 125.82604823112487}, \"b3369775dc\": {\"quality\": 0.49406504065040646, \"cost\": 0.017937791999999998, \"time\": 202.59222974777222}, \"b3b9205f60\": {\"quality\": 0.20133565621370497, \"cost\": 0.03923410000000001, \"time\": 184.9780117034912}, \"b3c56f0b3c\": {\"quality\": 0.594959349593496, \"cost\": 0.06256806000000001, \"time\": 168.73735642433167}, \"b3f20b706d\": {\"quality\": 0.6195121951219512, \"cost\": 0.0037520640000000003, \"time\": 106.53766713142394}, \"b4002173ee\": {\"quality\": 0.5188617886178861, \"cost\": 0.007600319999999999, \"time\": 109.14190998077393}, \"b46d382384\": {\"quality\": 0.7555284552845528, \"cost\": 0.12484462, \"time\": 146.6869523525238}, \"b4b2482ef9\": {\"quality\": 0.4543554006968641, \"cost\": 0.06673528000000001, \"time\": 163.60684385299683}, \"b4be043238\": {\"quality\": 0.3460394889663182, \"cost\": 0.08778872000000001, \"time\": 150.07519998550416}, \"b531bd0548\": {\"quality\": 0.3177584204413472, \"cost\": 0.08969832, \"time\": 202.44645614624022}, \"b56c312eda\": {\"quality\": 0.5277351916376307, \"cost\": 0.016686852, \"time\": 249.4414801120758}, \"b5e2b41c1c\": {\"quality\": 0.551684088269454, \"cost\": 0.009034968, \"time\": 169.00915660858155}, \"b61ce57a90\": {\"quality\": 0.36904761904761907, \"cost\": 0.039441384, \"time\": 192.418292427063}, \"b64ddb14f9\": {\"quality\": 0.3847967479674797, \"cost\": 0.02944528, \"time\": 824.0557322502136}, \"b67107a43e\": {\"quality\": 0.6195121951219512, \"cost\": 0.01083792, \"time\": 248.4702594280243}, \"b67720aa5c\": {\"quality\": 0.6195121951219512, \"cost\": 0.05931710800000001, \"time\": 146.00394463539124}, \"b682a23b89\": {\"quality\": 0.7337746806039489, \"cost\": 0.013234963999999998, \"time\": 213.58269662857055}, \"b69ef5add4\": {\"quality\": 0.6584204413472706, \"cost\": 0.07431707200000001, \"time\": 161.14033164978028}, \"b796b7ffd3\": {\"quality\": 0.3386178861788618, \"cost\": 0.014401015999999999, \"time\": 259.1940643310547}, \"b7a0083dc4\": {\"quality\": 0.6195121951219512, \"cost\": 0.061847876, \"time\": 214.91841106414796}, \"b7d0e8557f\": {\"quality\": 0.5686411149825784, \"cost\": 0.012575807999999999, \"time\": 247.72326970100403}, \"b8317a3a8c\": {\"quality\": 0.7867131242740998, \"cost\": 0.06423970400000001, \"time\": 181.56251420974732}, \"b8ab3d2f25\": {\"quality\": 0.5626016260162602, \"cost\": 0.015544804, \"time\": 184.08619875907897}, \"b8b569172f\": {\"quality\": 0.5490243902439025, \"cost\": 0.015828372, \"time\": 178.33178467750548}, \"b8f5ab44bb\": {\"quality\": 0.634959349593496, \"cost\": 0.07826424000000001, \"time\": 262.68980021476744}, \"b91e7fdb29\": {\"quality\": 0.6195121951219512, \"cost\": 0.012287124, \"time\": 227.29503026008607}, \"b932beaaa6\": {\"quality\": 0.544959349593496, \"cost\": 0.06720848000000001, \"time\": 186.7761948108673}, \"b9770c2261\": {\"quality\": 0.5645644599303136, \"cost\": 0.00541122, \"time\": 150.7996078491211}, \"b9bb1e6f8d\": {\"quality\": 0.6998373983739838, \"cost\": 0.018532544, \"time\": 244.14531807899476}, \"b9d0e8740c\": {\"quality\": 0.6134146341463415, \"cost\": 0.07111140800000001, \"time\": 214.80163626670839}, \"b9da208432\": {\"quality\": 0.641869918699187, \"cost\": 0.019034844000000002, \"time\": 232.56544876098633}, \"ba3223f6ac\": {\"quality\": 0.6195121951219512, \"cost\": 0.005706648, \"time\": 151.19898543357849}, \"bb13365175\": {\"quality\": 0.7763995354239257, \"cost\": 0.042266292000000004, \"time\": 224.2499403476715}, \"bb1b3a4d29\": {\"quality\": 0.8029384436701509, \"cost\": 0.064898518, \"time\": 256.5067723274231}, \"bb6536b0ab\": {\"quality\": 0.6195121951219512, \"cost\": 0.013670112000000002, \"time\": 159.04916138648986}, \"bbba9dd6ae\": {\"quality\": 0.5152032520325204, \"cost\": 0.008736126, \"time\": 125.51340088844299}, \"bbde69a1ae\": {\"quality\": 0.47317073170731705, \"cost\": 0.07075216, \"time\": 211.4701558113098}, \"bc29a0c0fe\": {\"quality\": 0.6923809523809524, \"cost\": 0.08504816000000001, \"time\": 152.53999376296997}, \"bc3d02f753\": {\"quality\": 0.6195121951219512, \"cost\": 0.0033908220000000004, \"time\": 108.17700786590576}, \"bc4c1fcc64\": {\"quality\": 0.25837398373983744, \"cost\": 0.008965720000000002, \"time\": 110.68981971740723}, \"bd30d27f62\": {\"quality\": 0.8004529616724738, \"cost\": 0.13683402800000002, \"time\": 179.2394115447998}, \"bd99b2fb21\": {\"quality\": 0.6292682926829268, \"cost\": 0.013529848, \"time\": 165.33139224052428}, \"bddc7d2a34\": {\"quality\": 0.7716144018583042, \"cost\": 0.06608894800000001, \"time\": 242.8191273212433}, \"be2ae88f70\": {\"quality\": 0.6195121951219512, \"cost\": 0.0070768079999999995, \"time\": 175.22385454177856}, \"be4740f38f\": {\"quality\": 0.5178048780487805, \"cost\": 0.018557584000000002, \"time\": 177.69371061325074}, \"bec0c6a95f\": {\"quality\": 0.31897793263646923, \"cost\": 0.08918554800000002, \"time\": 219.01660284996032}, \"bed888d4dc\": {\"quality\": 0.6195121951219512, \"cost\": 0.012997296, \"time\": 392.78799629211426}, \"bf45e407f6\": {\"quality\": 0.7567479674796748, \"cost\": 0.085539992, \"time\": 158.9132725715637}, \"bf5550f320\": {\"quality\": 0.6613821138211382, \"cost\": 0.070068868, \"time\": 233.04684481620788}, \"bf87e58322\": {\"quality\": 0.6940650406504065, \"cost\": 0.06185992, \"time\": 188.9820168018341}, \"bfed7670ed\": {\"quality\": 0.645609756097561, \"cost\": 0.014454772, \"time\": 144.87740097045898}, \"c0541e2220\": {\"quality\": 0.6195121951219512, \"cost\": 0.06049530400000001, \"time\": 134.89153051376343}, \"c0e10c0048\": {\"quality\": 0.6195121951219512, \"cost\": 0.06394020800000001, \"time\": 253.89145169258117}, \"c127509a7a\": {\"quality\": 0.5921951219512195, \"cost\": 0.02507367, \"time\": 171.8149597644806}, \"c13682c7c7\": {\"quality\": 0.6195121951219512, \"cost\": 0.016085024, \"time\": 252.46734075546266}, \"c13d6e78e9\": {\"quality\": 0.4615098722415795, \"cost\": 0.026846708000000004, \"time\": 158.4395161151886}, \"c14ff3144d\": {\"quality\": 0.6380487804878049, \"cost\": 0.002752848, \"time\": 124.53351817131042}, \"c1e42ac47b\": {\"quality\": 0.5833333333333334, \"cost\": 0.07545873600000001, \"time\": 234.8793641090393}, \"c2949aa902\": {\"quality\": 0.6728455284552846, \"cost\": 0.00916251, \"time\": 212.7397180557251}, \"c31e956b35\": {\"quality\": 0.6621718931475028, \"cost\": 0.014615412, \"time\": 238.76983637809752}, \"c36b525dde\": {\"quality\": 0.781869918699187, \"cost\": 0.00692364, \"time\": 160.97053718566895}, \"c38326e2bd\": {\"quality\": 0.4835772357723577, \"cost\": 0.07338262, \"time\": 231.5795979499817}, \"c3ec2cec59\": {\"quality\": 0.6329268292682928, \"cost\": 0.017786080000000003, \"time\": 169.43426485061644}, \"c44720575f\": {\"quality\": 0.5554355400696864, \"cost\": 0.035053448, \"time\": 162.69704794883728}, \"c48ecefab6\": {\"quality\": 0.6195121951219512, \"cost\": 0.06189563200000001, \"time\": 231.7577772140503}, \"c4a64eb40f\": {\"quality\": 0.5757375145180024, \"cost\": 0.087534268, \"time\": 171.7744441986084}, \"c4a80d19b3\": {\"quality\": 0.667862950058072, \"cost\": 0.017699555999999998, \"time\": 262.5382396697998}, \"c4c2826afd\": {\"quality\": 0.6195121951219512, \"cost\": 0.014996676, \"time\": 243.1779758453369}, \"c4c94a5527\": {\"quality\": 0.6195121951219512, \"cost\": 0.009552324000000001, \"time\": 171.8789219379425}, \"c4e75ee9ba\": {\"quality\": 0.6092334494773519, \"cost\": 0.14632424400000002, \"time\": 163.68729362487792}, \"c4f3e7665d\": {\"quality\": 0.6195121951219512, \"cost\": 0.006700248000000001, \"time\": 171.2033597946167}, \"c5471bef57\": {\"quality\": 0.6195121951219512, \"cost\": 0.06709224, \"time\": 360.6734181404114}, \"c54a408db7\": {\"quality\": 0.4271544715447154, \"cost\": 0.061197178000000005, \"time\": 205.84643664360047}, \"c59cc41335\": {\"quality\": 0.6195121951219512, \"cost\": 0.060401755999999994, \"time\": 227.63298444747926}, \"c5a0b065e0\": {\"quality\": 0.6585365853658537, \"cost\": 0.020608076000000003, \"time\": 100.40361194610595}, \"c5a16b834a\": {\"quality\": 0.6756097560975609, \"cost\": 0.020331156, \"time\": 377.4331964969635}, \"c5fbe2076f\": {\"quality\": 0.2873170731707317, \"cost\": 0.024696848, \"time\": 235.4668863296509}, \"c617370f6b\": {\"quality\": 0.6195121951219512, \"cost\": 0.05902978000000001, \"time\": 201.3127824783325}, \"c67f782c7f\": {\"quality\": 0.6860975609756098, \"cost\": 0.064923248, \"time\": 186.35708870887757}, \"c691a29c42\": {\"quality\": 0.5146341463414634, \"cost\": 0.02263088, \"time\": 342.815619802475}, \"c6a339987c\": {\"quality\": 0.6195121951219512, \"cost\": 0.014977467999999999, \"time\": 256.19790320396424}, \"c772ff3704\": {\"quality\": 0.5390243902439025, \"cost\": 0.012446268, \"time\": 256.34659972190855}, \"c7e3f348c2\": {\"quality\": 0.7056910569105692, \"cost\": 0.07413946400000002, \"time\": 226.76761050224303}, \"c823589ab6\": {\"quality\": 0.6195121951219512, \"cost\": 0.060011588000000005, \"time\": 169.34995718002318}, \"c82f834e85\": {\"quality\": 0.6195121951219512, \"cost\": 0.013014976000000001, \"time\": 360.72568283081057}, \"c85099881f\": {\"quality\": 0.6214634146341463, \"cost\": 0.019014336, \"time\": 91.83121838569642}, \"c935a33384\": {\"quality\": 0.47804878048780486, \"cost\": 0.021866696, \"time\": 201.4007071018219}, \"ca3177461f\": {\"quality\": 0.6390243902439025, \"cost\": 0.064102784, \"time\": 260.361168384552}, \"caa7c0bd6b\": {\"quality\": 0.614308943089431, \"cost\": 0.013427039999999998, \"time\": 201.50019497871398}, \"cac6b051e9\": {\"quality\": 0.6777119628339141, \"cost\": 0.012200891999999998, \"time\": 235.3216923236847}, \"cacb342f64\": {\"quality\": 0.3350406504065041, \"cost\": 0.09074681200000001, \"time\": 199.62914476394653}, \"cb9948679c\": {\"quality\": 0.6195121951219512, \"cost\": 0.060963451999999994, \"time\": 209.66846399307252}, \"cbb5eb0e74\": {\"quality\": 0.36753774680603957, \"cost\": 0.012385512, \"time\": 269.31641936302185}, \"cbc32cbeff\": {\"quality\": 0.09656213704994195, \"cost\": 0.012335669999999998, \"time\": 223.05522747039794}, \"cbd4461293\": {\"quality\": 0.4478861788617886, \"cost\": 0.037316864, \"time\": 1704.694543504715}, \"cbe2318045\": {\"quality\": 0.35097560975609754, \"cost\": 0.011905024, \"time\": 143.2994128704071}, \"cc886fe337\": {\"quality\": 0.6195121951219512, \"cost\": 0.008570712000000001, \"time\": 189.0461087703705}, \"cc9a6248a0\": {\"quality\": 0.8031823461091754, \"cost\": 0.06325050000000002, \"time\": 65.81817102432251}, \"ccb2335b3f\": {\"quality\": 0.30666666666666675, \"cost\": 0.066565, \"time\": 270.24665660858153}, \"ccdf03a55b\": {\"quality\": 0.554239256678281, \"cost\": 0.066687724, \"time\": 194.36894397735597}, \"ccf72745c1\": {\"quality\": 0.5723577235772358, \"cost\": 0.015334892, \"time\": 180.34684844017028}, \"cd1d418732\": {\"quality\": 0.6195121951219512, \"cost\": 0.007930735999999999, \"time\": 90.1477038860321}, \"cd23c79db1\": {\"quality\": 0.6195121951219512, \"cost\": 0.013173464000000001, \"time\": 219.91192269325256}, \"cd64fbfcd9\": {\"quality\": 0.6552845528455286, \"cost\": 0.010721148, \"time\": 215.86649370193481}, \"cd85a01e81\": {\"quality\": 0.6195121951219512, \"cost\": 0.066680636, \"time\": 309.61991782188414}, \"ce4bc5f348\": {\"quality\": 0.4926829268292683, \"cost\": 0.004973436, \"time\": 122.72588214874267}, \"ce980cf86f\": {\"quality\": 0.3678281068524971, \"cost\": 0.006999414, \"time\": 186.7628839492798}, \"ceae8b8bb9\": {\"quality\": 0.569965156794425, \"cost\": 0.040919136, \"time\": 278.74190187454224}, \"cecca90dd2\": {\"quality\": 0.7816376306620209, \"cost\": 0.009244559999999999, \"time\": 204.2295413017273}, \"cf9538faf0\": {\"quality\": 0.5823577235772358, \"cost\": 0.013121356, \"time\": 168.41699748039247}, \"cf9d2e224c\": {\"quality\": 0.5317073170731708, \"cost\": 0.008401064, \"time\": 129.19066014289857}, \"cfd36f3a8c\": {\"quality\": 0.5922764227642275, \"cost\": 0.026429034, \"time\": 197.97676906585693}, \"cffa29a6ef\": {\"quality\": 0.6403252032520326, \"cost\": 0.0011394740000000001, \"time\": 84.72708497047424}, \"d03596c3de\": {\"quality\": 0.6584552845528455, \"cost\": 0.017459960000000004, \"time\": 243.87012338638306}, \"d07b766487\": {\"quality\": 0.5414634146341464, \"cost\": 0.06639410400000001, \"time\": 248.754083442688}, \"d0a0a66d75\": {\"quality\": 0.4197560975609756, \"cost\": 0.037255556, \"time\": 155.2362714290619}, \"d0ce31134c\": {\"quality\": 0.6195121951219512, \"cost\": 0.07035569600000001, \"time\": 244.35629963874817}, \"d0f9633442\": {\"quality\": 0.6195121951219512, \"cost\": 0.010673088, \"time\": 174.68734726905822}, \"d216eab7d8\": {\"quality\": 0.3921138211382114, \"cost\": 0.015362728, \"time\": 177.72364864349365}, \"d266c19ac8\": {\"quality\": 0.5590592334494773, \"cost\": 0.010259069999999999, \"time\": 249.80076293945314}, \"d26a70179a\": {\"quality\": 0.6736701509872242, \"cost\": 0.072242652, \"time\": 281.9195426940918}, \"d2af24b59e\": {\"quality\": 0.37926829268292683, \"cost\": 0.012534632, \"time\": 162.15444717407226}, \"d2f2dd5cd4\": {\"quality\": 0.6195121951219512, \"cost\": 0.033032736, \"time\": 161.3020932674408}, \"d302278f85\": {\"quality\": 0.6195121951219512, \"cost\": 0.065508068, \"time\": 281.0260838031769}, \"d37dcaea30\": {\"quality\": 0.5957026713124275, \"cost\": 0.042292012000000004, \"time\": 254.16097497940063}, \"d3a2d50bd7\": {\"quality\": 0.5393263646922184, \"cost\": 0.008568708, \"time\": 180.23848376274108}, \"d3d4185487\": {\"quality\": 0.6195121951219512, \"cost\": 0.063731828, \"time\": 186.93449816703796}, \"d3db4cf84d\": {\"quality\": 0.6195121951219512, \"cost\": 0.007814196, \"time\": 126.7787591457367}, \"d402233b53\": {\"quality\": 0.7804761904761903, \"cost\": 0.04559879200000001, \"time\": 268.7110698223114}, \"d43fafa19e\": {\"quality\": 0.6195121951219512, \"cost\": 0.006752928, \"time\": 137.8223207473755}, \"d446a75eb7\": {\"quality\": 0.508432055749129, \"cost\": 0.07295704800000001, \"time\": 211.27834362983702}, \"d48ead13da\": {\"quality\": 0.6195121951219512, \"cost\": 0.010855976, \"time\": 194.97514882087708}, \"d5016f4538\": {\"quality\": 0.6195121951219512, \"cost\": 0.006847740000000001, \"time\": 154.1272620677948}, \"d55a3613b0\": {\"quality\": 0.7033565621370499, \"cost\": 0.09393361200000001, \"time\": 280.48901586532594}, \"d58036ba66\": {\"quality\": 0.6341463414634146, \"cost\": 0.012329896000000002, \"time\": 182.65342464447022}, \"d5a84c782e\": {\"quality\": 0.7512311265969802, \"cost\": 0.06159429000000001, \"time\": 238.61238617897033}, \"d5b2eef11c\": {\"quality\": 0.6305458768873403, \"cost\": 0.059769270000000006, \"time\": 148.21854801177977}, \"d6040140b9\": {\"quality\": 0.6772357723577236, \"cost\": 0.025150488000000006, \"time\": 293.03985538482664}, \"d65185c1a4\": {\"quality\": 0.5714750290360046, \"cost\": 0.011127215999999999, \"time\": 211.66472873687744}, \"d667351f33\": {\"quality\": 0.6195121951219512, \"cost\": 0.07187473600000001, \"time\": 284.58656883239746}, \"d6bd3b66ba\": {\"quality\": 0.6195121951219512, \"cost\": 0.0062973000000000005, \"time\": 179.81442737579346}, \"d6c4e48eeb\": {\"quality\": 0.3821138211382114, \"cost\": 0.013097992, \"time\": 185.88441767692566}, \"d6cbf265ee\": {\"quality\": 0.553170731707317, \"cost\": 0.027157003999999995, \"time\": 2382.298054933548}, \"d705447fd7\": {\"quality\": 0.6195121951219512, \"cost\": 0.06927182000000001, \"time\": 274.04458661079406}, \"d73a9aab4e\": {\"quality\": 0.6195121951219512, \"cost\": 0.00601782, \"time\": 119.8721248626709}, \"d752c30d07\": {\"quality\": 0.6491869918699187, \"cost\": 0.029268951999999997, \"time\": 187.99695973396302}, \"d782682359\": {\"quality\": 0.48292682926829267, \"cost\": 0.070581532, \"time\": 240.62954745292663}, \"d7c0972014\": {\"quality\": 0.46747967479674796, \"cost\": 0.014235999999999999, \"time\": 146.19986548423768}, \"d867525748\": {\"quality\": 0.2914634146341463, \"cost\": 0.031817048, \"time\": 176.6470585823059}, \"d87eb775da\": {\"quality\": 0.5716260162601626, \"cost\": 0.011791225999999998, \"time\": 171.5166805744171}, \"d8bab6c09b\": {\"quality\": 0.5560975609756098, \"cost\": 0.0201636, \"time\": 262.4358974933624}, \"d8bcac36e8\": {\"quality\": 0.6195121951219512, \"cost\": 0.014129348, \"time\": 178.82634310722352}, \"d8eadc0190\": {\"quality\": 0.6644715447154471, \"cost\": 0.08632676000000002, \"time\": 188.2232988357544}, \"d96677d8d4\": {\"quality\": 0.7195005807200929, \"cost\": 0.010380108, \"time\": 202.3421525001526}, \"d98f22270e\": {\"quality\": 0.46382113821138216, \"cost\": 0.06577788, \"time\": 195.36387667655944}, \"d9e2bb21a3\": {\"quality\": 0.5120789779326365, \"cost\": 0.011266182, \"time\": 204.83109889030456}, \"da95deeb20\": {\"quality\": 0.3967479674796748, \"cost\": 0.05949913, \"time\": 146.4172206401825}, \"daaadadcc9\": {\"quality\": 0.6085365853658538, \"cost\": 0.014349220000000001, \"time\": 236.59604306221007}, \"daf855e065\": {\"quality\": 0.7283739837398373, \"cost\": 0.01084773, \"time\": 254.38643417358398}, \"db00594832\": {\"quality\": 0.7669570267131242, \"cost\": 0.082393492, \"time\": 131.31271381378173}, \"db19e677c4\": {\"quality\": 0.816341463414634, \"cost\": 0.08040591200000001, \"time\": 186.9021366119385}, \"db3c035639\": {\"quality\": 0.6329268292682927, \"cost\": 0.07513583600000001, \"time\": 251.00368614196776}, \"db41487005\": {\"quality\": 0.6195121951219512, \"cost\": 0.05802746000000001, \"time\": 196.43507223129274}, \"db6a7482fd\": {\"quality\": 0.6803716608594658, \"cost\": 0.056941420000000006, \"time\": 138.47659482955933}, \"db9060cd27\": {\"quality\": 0.42195121951219516, \"cost\": 0.027881287999999997, \"time\": 1896.3412520885468}, \"dbce95a072\": {\"quality\": 0.5060975609756098, \"cost\": 0.012663032000000001, \"time\": 179.2968252182007}, \"dc195abe5e\": {\"quality\": 0.31232288037166084, \"cost\": 0.062047202, \"time\": 256.8396565437317}, \"dc3f4b7138\": {\"quality\": 0.6195121951219512, \"cost\": 0.05797826800000001, \"time\": 188.94316940307618}, \"dc66bccb1c\": {\"quality\": 0.3678048780487805, \"cost\": 0.01096708, \"time\": 168.7502061367035}, \"dc90065dea\": {\"quality\": 0.36016260162601627, \"cost\": 0.014289246, \"time\": 199.27637152671815}, \"dd0d70fedd\": {\"quality\": 0.7897677119628339, \"cost\": 0.005333369999999999, \"time\": 160.19438481330872}, \"dddc76b3ca\": {\"quality\": 0.32487804878048776, \"cost\": 0.06413708, \"time\": 203.57222208976745}, \"de18bf45e1\": {\"quality\": 0.5365040650406504, \"cost\": 0.009448855999999999, \"time\": 966.3698208808898}, \"de1e56370f\": {\"quality\": 0.6371544715447155, \"cost\": 0.007415147999999999, \"time\": 141.48761720657347}, \"df2160ecc8\": {\"quality\": 0.4975609756097561, \"cost\": 0.06514112000000001, \"time\": 184.16321535110472}, \"dfda94bd2a\": {\"quality\": 0.6195121951219512, \"cost\": 0.0029004839999999996, \"time\": 111.35115175247192}, \"dff452a9ca\": {\"quality\": 0.6195121951219512, \"cost\": 0.00818748, \"time\": 128.0037736415863}, \"e09f75d9d6\": {\"quality\": 0.5954123112659697, \"cost\": 0.07994701400000001, \"time\": 174.90588331222534}, \"e0b6a99753\": {\"quality\": 0.6808943089430894, \"cost\": 0.022907588, \"time\": 75.86744494438172}, \"e0cf6587a7\": {\"quality\": 0.38739837398373983, \"cost\": 0.06783974000000001, \"time\": 224.5930316925049}, \"e0de4a5929\": {\"quality\": 0.28844367015098726, \"cost\": 0.070857196, \"time\": 237.39370694160462}, \"e1356fb426\": {\"quality\": 0.563054587688734, \"cost\": 0.06065425, \"time\": 185.1619038105011}, \"e20ba014a1\": {\"quality\": 0.6341463414634146, \"cost\": 0.022406492, \"time\": 231.54046459197997}, \"e21806e3bc\": {\"quality\": 0.6686991869918699, \"cost\": 0.030775240000000002, \"time\": 153.30845246315002}, \"e24e97564c\": {\"quality\": 0.5788269454123112, \"cost\": 0.065976868, \"time\": 234.19773464202882}, \"e2673c1ec8\": {\"quality\": 0.5588617886178862, \"cost\": 0.07093568800000001, \"time\": 268.0965113162994}, \"e26c7bfbdb\": {\"quality\": 0.526829268292683, \"cost\": 0.010329528, \"time\": 162.32453393936157}, \"e2f9980b06\": {\"quality\": 0.8120905923344948, \"cost\": 0.041606056, \"time\": 251.1425283432007}, \"e3445f7632\": {\"quality\": 0.7865737514518002, \"cost\": 0.02273798, \"time\": 124.58966102600098}, \"e35e5f81a7\": {\"quality\": 0.6195121951219512, \"cost\": 0.012546798000000001, \"time\": 182.4151572227478}, \"e376ac53e7\": {\"quality\": 0.5323577235772358, \"cost\": 0.031615856, \"time\": 169.9604299068451}, \"e3d8bb56da\": {\"quality\": 0.7227758420441346, \"cost\": 0.03464479, \"time\": 173.92777223587035}, \"e3df4cf041\": {\"quality\": 0.6195121951219512, \"cost\": 0.013172260000000002, \"time\": 228.6481415748596}, \"e47dc3abca\": {\"quality\": 0.5421602787456445, \"cost\": 0.01662284, \"time\": 196.10405125617982}, \"e4b9d4fb41\": {\"quality\": 0.6195121951219512, \"cost\": 0.013018407999999999, \"time\": 231.83416152000427}, \"e510bda989\": {\"quality\": 0.4415447154471545, \"cost\": 0.12802820799999998, \"time\": 1276.7408336162566}, \"e517cd2222\": {\"quality\": 0.3045528455284553, \"cost\": 0.020794063999999998, \"time\": 1299.5594879627229}, \"e51b01f418\": {\"quality\": 0.6195121951219512, \"cost\": 0.011492568, \"time\": 202.7894808292389}, \"e520dfae5b\": {\"quality\": 0.6439024390243903, \"cost\": 0.011328468, \"time\": 263.53743448257444}, \"e521c9b7e4\": {\"quality\": 0.7932520325203252, \"cost\": 0.06690654, \"time\": 156.73078799247742}, \"e54097ad5d\": {\"quality\": 0.5923344947735192, \"cost\": 0.062158128, \"time\": 209.91472067832947}, \"e56a16ca66\": {\"quality\": 0.4848315911730546, \"cost\": 0.004748472, \"time\": 137.46008925437928}, \"e5c4abf7ce\": {\"quality\": 0.35598141695702673, \"cost\": 0.06409228000000002, \"time\": 256.3565216064453}, \"e5d4689312\": {\"quality\": 0.6195121951219512, \"cost\": 0.06653434, \"time\": 235.4001616001129}, \"e62a7b27ae\": {\"quality\": 0.6195121951219512, \"cost\": 0.035626552, \"time\": 195.45834450721742}, \"e6a7aff3bc\": {\"quality\": 0.6195121951219512, \"cost\": 0.06874828800000002, \"time\": 295.3631275177002}, \"e736999157\": {\"quality\": 0.6195121951219512, \"cost\": 0.06437495600000001, \"time\": 288.5091665744782}, \"e7517a8ce0\": {\"quality\": 0.8125435540069686, \"cost\": 0.08348902, \"time\": 200.60669808387757}, \"e7520ca5ac\": {\"quality\": 0.36312427409988385, \"cost\": 0.06479258000000002, \"time\": 261.0504195690155}, \"e7e94ab7a5\": {\"quality\": 0.591869918699187, \"cost\": 0.019925159999999997, \"time\": 2488.357094335556}, \"e887ddf5cc\": {\"quality\": 0.48739837398373986, \"cost\": 0.09377338400000002, \"time\": 253.3760078907013}, \"e94fb5a295\": {\"quality\": 0.6195121951219512, \"cost\": 0.07366761200000001, \"time\": 223.6755922794342}, \"ea6ecc5653\": {\"quality\": 0.6166666666666667, \"cost\": 0.010067135999999999, \"time\": 190.92216954231262}, \"ea8bcb3ae2\": {\"quality\": 0.45772357723577234, \"cost\": 0.022977280000000003, \"time\": 212.54974527359008}, \"ebbe8b6c4f\": {\"quality\": 0.6195121951219512, \"cost\": 0.009877384, \"time\": 160.62470216751097}, \"ebdf3abff2\": {\"quality\": 0.686260162601626, \"cost\": 0.019338396, \"time\": 179.08990707397462}, \"ec55dba809\": {\"quality\": 0.4276422764227642, \"cost\": 0.068014448, \"time\": 230.82128653526306}, \"ecb5f78f37\": {\"quality\": 0.6195121951219512, \"cost\": 0.017395611999999998, \"time\": 249.2554892539978}, \"ecda3d74cb\": {\"quality\": 0.6195121951219512, \"cost\": 0.061945256000000004, \"time\": 200.0551459789276}, \"ece10c0388\": {\"quality\": 0.7655168408826946, \"cost\": 0.129639792, \"time\": 172.15098700523376}, \"ed6b5480a5\": {\"quality\": 0.47195121951219515, \"cost\": 0.006612492, \"time\": 117.65485558509826}, \"eda630dc85\": {\"quality\": 0.6290127758420442, \"cost\": 0.013249908000000001, \"time\": 202.01231064796448}, \"edaaee5ed4\": {\"quality\": 0.5414634146341464, \"cost\": 0.011020607999999998, \"time\": 124.82250752449036}, \"edb2b764aa\": {\"quality\": 0.6329268292682928, \"cost\": 0.018810900000000002, \"time\": 160.87249393463134}, \"edc52339db\": {\"quality\": 0.6195121951219512, \"cost\": 0.0179609, \"time\": 246.9092752456665}, \"ede7071775\": {\"quality\": 0.7035423925667827, \"cost\": 0.13207611000000002, \"time\": 200.41918315887452}, \"ee46042c5d\": {\"quality\": 0.6195121951219512, \"cost\": 0.016286580000000002, \"time\": 244.87669949531556}, \"ee68c51f73\": {\"quality\": 0.6029500580720093, \"cost\": 0.07541250000000001, \"time\": 109.02409811019898}, \"ee7b726747\": {\"quality\": 0.7833797909407665, \"cost\": 0.039892712, \"time\": 251.92134594917297}, \"eec5f32da9\": {\"quality\": 0.81595818815331, \"cost\": 0.0879383, \"time\": 286.01010518074037}, \"eed40e4378\": {\"quality\": 0.6613821138211382, \"cost\": 0.06929234, \"time\": 282.0774456501007}, \"eef12d478b\": {\"quality\": 0.46260162601626015, \"cost\": 0.009616956, \"time\": 204.5144693851471}, \"ef37b3e0be\": {\"quality\": 0.6195121951219512, \"cost\": 0.012494243999999998, \"time\": 257.47455110549924}, \"ef43d497f1\": {\"quality\": 0.40569105691056906, \"cost\": 0.016996992, \"time\": 246.4581404685974}, \"ef4d4c4a62\": {\"quality\": 0.4484552845528455, \"cost\": 0.015539336, \"time\": 182.2000419616699}, \"ef9a651425\": {\"quality\": 0.7040650406504065, \"cost\": 0.07579404, \"time\": 250.92651562690736}, \"f0655621af\": {\"quality\": 0.4265853658536585, \"cost\": 0.15761675999999997, \"time\": 2013.6334864139558}, \"f076b4c9ae\": {\"quality\": 0.7834262485481996, \"cost\": 0.015863708, \"time\": 187.7628330230713}, \"f11eddb4ed\": {\"quality\": 0.20284552845528453, \"cost\": 0.016428986, \"time\": 237.75844135284424}, \"f1408da253\": {\"quality\": 0.6967479674796748, \"cost\": 0.014442047999999999, \"time\": 249.8467248916626}, \"f18cf41929\": {\"quality\": 0.6195121951219512, \"cost\": 0.003961836, \"time\": 117.34907512664795}, \"f1aa0b0b42\": {\"quality\": 0.5720325203252032, \"cost\": 0.043219422, \"time\": 146.36151003837585}, \"f1bda127f6\": {\"quality\": 0.46178861788617886, \"cost\": 0.017929852000000003, \"time\": 197.9198618412018}, \"f1f373e58e\": {\"quality\": 0.7831010452961673, \"cost\": 0.09002725200000002, \"time\": 185.47337765693663}, \"f2a2e91541\": {\"quality\": 0.4859117305458769, \"cost\": 0.012855611999999999, \"time\": 232.05357608795165}, \"f2c04ed1c8\": {\"quality\": 0.405609756097561, \"cost\": 0.0067632479999999995, \"time\": 126.54541292190552}, \"f2cf5db12d\": {\"quality\": 0.19959349593495934, \"cost\": 0.013177552, \"time\": 168.4030219078064}, \"f366c0dd10\": {\"quality\": 0.6195121951219512, \"cost\": 0.06272198, \"time\": 196.8466109275818}, \"f4303a5b4f\": {\"quality\": 0.6103252032520325, \"cost\": 0.09301452800000001, \"time\": 231.81033272743224}, \"f437481e3b\": {\"quality\": 0.5909756097560975, \"cost\": 0.019717612, \"time\": 227.29155945777893}, \"f4bc6b63a7\": {\"quality\": 0.571788617886179, \"cost\": 0.12058799000000002, \"time\": 197.6338578224182}, \"f4dc556633\": {\"quality\": 0.6195121951219512, \"cost\": 0.06797158, \"time\": 289.321187877655}, \"f4deb72db6\": {\"quality\": 0.6195121951219512, \"cost\": 0.0105891, \"time\": 200.89887857437134}, \"f4ef2b9c33\": {\"quality\": 0.5439837398373983, \"cost\": 0.03817632000000001, \"time\": 213.8931649684906}, \"f566d6d6a1\": {\"quality\": 0.4036585365853659, \"cost\": 0.062300763999999995, \"time\": 201.07639598846436}, \"f5b9a94dcc\": {\"quality\": 0.6146341463414634, \"cost\": 0.04594065199999999, \"time\": 1847.3314466953277}, \"f5c27e7172\": {\"quality\": 0.6195121951219512, \"cost\": 0.007075463999999999, \"time\": 232.92631516456603}, \"f5e53d963b\": {\"quality\": 0.6195121951219512, \"cost\": 0.004926876000000001, \"time\": 146.53346581459044}, \"f614235c15\": {\"quality\": 0.546829268292683, \"cost\": 0.019139696, \"time\": 1690.8127262592316}, \"f6546149e3\": {\"quality\": 0.7115214866434378, \"cost\": 0.08226788, \"time\": 214.23944363594055}, \"f74ec023e4\": {\"quality\": 0.6195121951219512, \"cost\": 0.016337988, \"time\": 300.67309379577637}, \"f7b048bd54\": {\"quality\": 0.46829268292682935, \"cost\": 0.016687760000000003, \"time\": 214.77183957099913}, \"f7c4df993e\": {\"quality\": 0.6378513356562137, \"cost\": 0.01224465, \"time\": 182.30367503166198}, \"f854533145\": {\"quality\": 0.6585365853658537, \"cost\": 0.017361696, \"time\": 270.88364191055297}, \"f89b8a1930\": {\"quality\": 0.8136236933797909, \"cost\": 0.040442136000000004, \"time\": 276.7925349235535}, \"f93d9a2693\": {\"quality\": 0.37878048780487805, \"cost\": 0.008193323999999998, \"time\": 204.3297384738922}, \"f97d91a249\": {\"quality\": 0.3878048780487805, \"cost\": 0.06316157600000001, \"time\": 184.88821873664855}, \"f99096d89c\": {\"quality\": 0.6195121951219512, \"cost\": 0.014407992, \"time\": 267.10197510719297}, \"f9e8e221f3\": {\"quality\": 0.39916376306620205, \"cost\": 0.010630103999999998, \"time\": 213.8131926059723}, \"fa38879eab\": {\"quality\": 0.6504065040650406, \"cost\": 0.021828032000000004, \"time\": 295.17917666435244}, \"fa71111570\": {\"quality\": 0.5853658536585366, \"cost\": 0.069471068, \"time\": 280.6139543056488}, \"fa7882d46b\": {\"quality\": 0.34783972125435536, \"cost\": 0.016001058000000002, \"time\": 247.1370331764221}, \"fa906520d1\": {\"quality\": 0.6655981416957026, \"cost\": 0.018073107999999997, \"time\": 259.80645098686216}, \"faabebaa30\": {\"quality\": 0.22723577235772358, \"cost\": 0.015047418000000002, \"time\": 238.21279168128967}, \"fb0339a7d0\": {\"quality\": 0.7656794425087108, \"cost\": 0.021738743999999997, \"time\": 273.6737798213959}, \"fb216ad6b3\": {\"quality\": 0.3592682926829268, \"cost\": 0.05636392800000001, \"time\": 978.5727415084839}, \"fb6216880a\": {\"quality\": 0.6195121951219512, \"cost\": 0.016447008, \"time\": 279.96212286949157}, \"fba499b89d\": {\"quality\": 0.6195121951219512, \"cost\": 0.06753407200000001, \"time\": 276.17343015670775}, \"fbc010a368\": {\"quality\": 0.6195121951219512, \"cost\": 0.06443590400000002, \"time\": 217.46284394264222}, \"fbc02e2e07\": {\"quality\": 0.6195121951219512, \"cost\": 0.061001164, \"time\": 259.36248960494993}, \"fbd6c45271\": {\"quality\": 0.24020905923344946, \"cost\": 0.015250424000000002, \"time\": 285.5677849769592}, \"fc0a156e16\": {\"quality\": 0.7864692218350754, \"cost\": 0.034151164, \"time\": 244.54232273101806}, \"fc1fd5bf54\": {\"quality\": 0.6195121951219512, \"cost\": 0.010393596, \"time\": 302.0887975215912}, \"fc6967a75b\": {\"quality\": 0.6195121951219512, \"cost\": 0.06284471200000001, \"time\": 293.1112622261047}, \"fc73c3b0fa\": {\"quality\": 0.6592682926829267, \"cost\": 0.025536140000000002, \"time\": 98.14458861351014}, \"fce38334b2\": {\"quality\": 0.6195121951219512, \"cost\": 0.01724038, \"time\": 267.3305795669556}, \"fce5fca128\": {\"quality\": 0.4260278745644599, \"cost\": 0.066997868, \"time\": 262.8612917900085}, \"fd0709359e\": {\"quality\": 0.6446341463414634, \"cost\": 0.001408038, \"time\": 79.83381242752075}, \"fd1f809d64\": {\"quality\": 0.8097560975609757, \"cost\": 0.07385033799999999, \"time\": 213.83500204086303}, \"fd2c994a9d\": {\"quality\": 0.6390243902439025, \"cost\": 0.06503392399999999, \"time\": 284.80486459732055}, \"fddccfbf94\": {\"quality\": 0.46829268292682935, \"cost\": 0.014656296000000003, \"time\": 217.74681878089905}, \"fe7fa741b4\": {\"quality\": 0.48540069686411147, \"cost\": 0.034931979999999994, \"time\": 209.350274848938}, \"fe9e1fec71\": {\"quality\": 0.5036585365853659, \"cost\": 0.067148512, \"time\": 212.58801732063293}, \"fea4734c09\": {\"quality\": 0.4943902439024391, \"cost\": 0.008797415999999999, \"time\": 129.66075100898743}, \"fef1ca27fa\": {\"quality\": 0.4926829268292683, \"cost\": 0.020025176, \"time\": 248.60985913276673}, \"ff11cb6a7a\": {\"quality\": 0.39739837398373984, \"cost\": 0.0197844, \"time\": 228.27161712646483}, \"ff171e34e2\": {\"quality\": 0.6195121951219512, \"cost\": 0.007519518, \"time\": 156.71968116760254}, \"ff1c958e21\": {\"quality\": 0.6341463414634146, \"cost\": 0.0048062339999999995, \"time\": 124.33939328193665}, \"ff8df4ace9\": {\"quality\": 0.5845528455284553, \"cost\": 0.0622996, \"time\": 187.53624334335328}, \"ff8e68049a\": {\"quality\": 0.28821138211382114, \"cost\": 0.010620372, \"time\": 159.36370429992675}}" }, { "path": "abacus-research/cuad_data_loader.py", "content": "\"\"\"\nShared CUAD data loading utilities to replace HuggingFace datasets.\nAll CUAD scripts should import from this module.\n\"\"\"\n\nimport json\nimport os\n\nimport numpy as np\n\n# Default data directory\nDEFAULT_DATA_DIR = \"cuad-data\"\n\ndef load_cuad_data(split=\"test\", data_dir=None):\n \"\"\"\n Load CUAD dataset from local JSON files.\n \n Args:\n split: \"train\" or \"test\"\n data_dir: Directory containing CUAD JSON files (default: \"cuad-data\")\n \n Returns:\n List of dictionaries with CUAD data in flat format\n \"\"\"\n if data_dir is None:\n data_dir = DEFAULT_DATA_DIR\n \n if split == \"train\":\n file_path = os.path.join(data_dir, \"train_separate_questions.json\")\n else:\n file_path = os.path.join(data_dir, \"test.json\")\n \n if not os.path.exists(file_path):\n raise FileNotFoundError(\n f\"CUAD data file not found at {file_path}. \"\n f\"Please run 'python setup_cuad_data.py' first to download the data.\"\n )\n \n with open(file_path) as f:\n raw_data = json.load(f)\n\n # Convert to flat format\n dataset = []\n for article in raw_data[\"data\"]:\n title = article.get(\"title\", \"\").strip()\n for paragraph in article[\"paragraphs\"]:\n context = paragraph[\"context\"].strip()\n for qa in paragraph[\"qas\"]:\n dataset.append({\n \"id\": qa[\"id\"],\n \"title\": title,\n \"context\": context,\n \"question\": qa[\"question\"].strip(),\n \"answers\": qa.get(\"answers\", [])\n })\n \n return dataset\n\n\ndef get_unique_contracts(dataset):\n \"\"\"Get list of unique contract titles from dataset.\"\"\"\n contract_titles = []\n for row in dataset:\n if row[\"title\"] not in contract_titles:\n contract_titles.append(row[\"title\"])\n return contract_titles\n\n\ndef filter_by_contracts(dataset, contract_titles):\n \"\"\"Filter dataset to only include specified contracts.\"\"\"\n return [row for row in dataset if row[\"title\"] in contract_titles]\n\n\ndef sample_contracts(dataset, num_contracts, seed=42):\n \"\"\"\n Sample a subset of contracts from the dataset.\n \n Args:\n dataset: CUAD dataset\n num_contracts: Number of contracts to sample\n seed: Random seed for reproducibility\n \n Returns:\n Filtered dataset with only the sampled contracts\n \"\"\"\n contract_titles = get_unique_contracts(dataset)\n \n # Shuffle and sample\n rng = np.random.default_rng(seed=seed)\n rng.shuffle(contract_titles)\n sampled_titles = contract_titles[:num_contracts]\n \n return filter_by_contracts(dataset, sampled_titles), sampled_titles" }, { "path": "abacus-research/download_embeddings_and_mmqa.sh", "content": "#!/bin/bash\n\nwget -nc https://palimpzest-workloads.s3.us-east-1.amazonaws.com/abacus-data.tar.gz\ntar -xzf abacus-data.tar.gz\n\n" }, { "path": "abacus-research/helper-scripts/biodex-gen-index.py", "content": "import os\nimport time\n\nimport chromadb\nimport chromadb.utils.embedding_functions as embedding_functions\nimport numpy as np\nfrom openai import OpenAI\nfrom tqdm import tqdm\n\n# NOTE: this script is meant to be run from the root of the repository\nif __name__ == \"__main__\":\n # initialize openai client\n openai_client = OpenAI()\n\n # load reaction terms\n reaction_terms = []\n with open(\"testdata/reaction_terms.txt\") as f:\n for line in f:\n reaction_terms.append(line.strip())\n\n # create directory for embeddings\n os.makedirs(\"testdata/reaction-term-embeddings/\", exist_ok=True)\n\n # generate embeddings in batches of 1000 at a time\n indices = np.linspace(0, len(reaction_terms), len(reaction_terms)//1000, dtype=int)\n total_embeds = len(indices)\n print(f\"Generating {total_embeds} embeddings...\")\n gen_indices = []\n for iter_idx, start_idx in tqdm(enumerate(indices), total=total_embeds):\n # check if embedding needs to be computed\n end_idx = indices[iter_idx + 1] if iter_idx + 1 < len(indices) else None\n filename = f\"testdata/reaction-term-embeddings/{start_idx}_{end_idx}.npy\"\n if end_idx is not None and not os.path.exists(filename):\n # generate embeddings\n batch = reaction_terms[start_idx:end_idx]\n resp = openai_client.embeddings.create(input=batch, model=\"text-embedding-3-small\")\n embeddings = [item.embedding for item in resp.data]\n\n # save embeddings to disk\n with open(filename, \"wb\") as f:\n np.save(f, np.array(embeddings))\n\n gen_indices.append((start_idx, end_idx))\n time.sleep(1)\n print(\"Done generating embeddings.\")\n\n # initialize chroma client\n chroma_client = chromadb.PersistentClient(\".chroma-biodex\")\n\n # initialize embedding function\n openai_ef = embedding_functions.OpenAIEmbeddingFunction(\n api_key=os.environ[\"OPENAI_API_KEY\"],\n model_name=\"text-embedding-3-small\"\n )\n\n # create a collection\n collection = chroma_client.get_or_create_collection(\n name=\"biodex-reaction-terms\",\n embedding_function=openai_ef,\n metadata={\"hnsw:space\": \"cosine\"},\n )\n\n # insert documents in batches\n total_inserts = len(gen_indices)\n print(f\"Inserting {total_inserts} batches into the collection...\")\n for start_idx, end_idx in tqdm(gen_indices, total=total_inserts):\n embeddings = np.load(f\"testdata/reaction-term-embeddings/{start_idx}_{end_idx}.npy\")\n collection.add(\n documents=reaction_terms[start_idx:end_idx],\n embeddings=embeddings.tolist(),\n ids=[f\"id{idx}\" for idx in range(start_idx, end_idx)]\n )\n" }, { "path": "abacus-research/helper-scripts/generate-prior-stats-biodex-first-convert.py", "content": "\"\"\"\nNOTE: this script worked with the tag `abacus-paper-experiments` but is no longer compatible with the main branch.\n\"\"\"\nimport argparse\nimport json\nimport os\nimport time\n\nimport datasets\n\n# from ragatouille import RAGPretrainedModel\nimport palimpzest as pz\nfrom palimpzest.constants import Model\n\nbiodex_entry_cols = [\n {\"name\": \"pmid\", \"type\": str, \"desc\": \"The PubMed ID of the medical paper\"},\n {\"name\": \"title\", \"type\": str, \"desc\": \"The title of the medical paper\"},\n {\"name\": \"abstract\", \"type\": str, \"desc\": \"The abstract of the medical paper\"},\n {\"name\": \"fulltext\", \"type\": str, \"desc\": \"The full text of the medical paper, which contains information relevant for creating a drug safety report.\"},\n]\n\nbiodex_reactions_cols = [\n {\"name\": \"reactions\", \"type\": list[str], \"desc\": \"The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible.\"},\n]\n\nclass BiodexDataset(pz.IterDataset):\n def __init__(\n self,\n rp_at_k: int = 5,\n num_samples: int = 5,\n split: str = \"test\",\n shuffle: bool = False,\n seed: int = 42,\n ):\n super().__init__(id=f\"biodex-{split}\", schema=biodex_entry_cols)\n\n self.dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=split).to_pandas()\n if shuffle:\n self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient=\"records\")\n else:\n self.dataset = self.dataset.to_dict(orient=\"records\")[:num_samples]\n\n self.rp_at_k = rp_at_k\n self.num_samples = num_samples\n self.shuffle = shuffle\n self.seed = seed\n self.split = split\n\n def compute_label(self, entry: dict) -> dict:\n \"\"\"Compute the label for a BioDEX report given its entry in the dataset.\"\"\"\n reactions_lst = [\n reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n for reaction in entry[\"reactions\"].split(\",\")\n ]\n label_dict = {\"reactions\": reactions_lst}\n return label_dict\n\n @staticmethod\n def term_recall(preds: list | None, targets: list):\n if preds is None:\n return 0.0\n\n try:\n # normalize terms in each list\n pred_terms = set([\n term.strip()\n for pred in preds\n for term in pred.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n target_terms = ([\n term.strip()\n for target in targets\n for term in target.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n\n # compute term recall and return\n intersect = pred_terms.intersection(target_terms)\n term_recall = len(intersect) / len(target_terms)\n\n return term_recall\n\n except Exception:\n os.makedirs(\"term-recall-eval-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"term-recall-eval-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def __len__(self):\n return len(self.dataset)\n\n def __getitem__(self, idx: int):\n # get entry\n entry = self.dataset[idx]\n\n # get input fields\n pmid = entry[\"pmid\"]\n title = entry[\"title\"]\n abstract = entry[\"abstract\"]\n fulltext = entry[\"fulltext\"]\n\n # create item with fields\n item = {\"fields\": {}, \"labels\": {}, \"score_fn\": {}}\n item[\"fields\"][\"pmid\"] = pmid\n item[\"fields\"][\"title\"] = title\n item[\"fields\"][\"abstract\"] = abstract\n item[\"fields\"][\"fulltext\"] = fulltext\n\n if self.split == \"train\":\n # add label info\n item[\"labels\"] = self.compute_label(entry)\n\n # add scoring functions for list fields\n item[\"score_fn\"][\"reactions\"] = BiodexDataset.term_recall\n\n return item\n\n\nif __name__ == \"__main__\":\n # parse arguments\n parser = argparse.ArgumentParser(description=\"Run a simple demo\")\n parser.add_argument(\"--verbose\", default=False, action=\"store_true\", help=\"Print verbose output\")\n parser.add_argument(\"--progress\", default=True, action=\"store_true\", help=\"Print progress output\")\n args = parser.parse_args()\n\n # create directory for profiling data\n os.makedirs(\"priors-data\", exist_ok=True)\n\n verbose = args.verbose\n progress = args.progress\n seed = 123 # NOTE: unique to cascades run\n execution_strategy = \"parallel\"\n sentinel_execution_strategy = \"all\"\n optimizer_strategy = \"pareto\"\n exp_name = f\"biodex-priors-{optimizer_strategy}-seed{seed}-cascades\" # NOTE: unique to cascades run\n\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n # create data source\n dataset = BiodexDataset(\n split=\"test\",\n num_samples=1,\n shuffle=True,\n seed=seed,\n )\n\n # create validation data source\n train_dataset = BiodexDataset(\n split=\"train\",\n num_samples=5,\n shuffle=True,\n seed=seed,\n )\n\n # construct plan\n plan = dataset\n plan = plan.sem_add_columns(biodex_reactions_cols)\n\n # only use final op quality\n use_final_op_quality = True\n\n # execute pz plan\n config = pz.QueryProcessorConfig(\n optimizer_strategy=optimizer_strategy,\n sentinel_execution_strategy=sentinel_execution_strategy,\n execution_strategy=execution_strategy,\n use_final_op_quality=use_final_op_quality,\n max_workers=64,\n verbose=verbose,\n available_models=[ # NOTE: unique to cascades run\n # Model.GPT_4o,\n Model.GPT_4o_MINI,\n Model.LLAMA3_2_3B,\n Model.LLAMA3_1_8B,\n Model.LLAMA3_3_70B,\n Model.LLAMA3_2_90B_V,\n # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments`\n # Model.DEEPSEEK_V3,\n Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B,\n ],\n allow_bonded_query=True,\n allow_critic=True,\n allow_mixtures=True,\n allow_rag_reduction=True,\n progress=progress,\n k=-1,\n j=-1,\n sample_budget=5*1014,\n seed=seed,\n exp_name=exp_name,\n )\n\n data_record_collection = plan.run(config=config, train_dataset=train_dataset, validator=pz.Validator())\n\n print(data_record_collection.to_df())\n data_record_collection.to_df().to_csv(f\"priors-data/{exp_name}-output.csv\", index=False)\n\n # create filepaths for records and stats\n records_path = f\"priors-data/{exp_name}-records.json\"\n stats_path = f\"priors-data/{exp_name}-profiling.json\"\n\n # save record outputs\n record_jsons = []\n for record in data_record_collection:\n record_dict = record.to_dict()\n record_dict = {\n k: v\n for k, v in record_dict.items()\n if k in [\"pmid\", \"reactions\"]\n }\n record_jsons.append(record_dict)\n\n with open(records_path, \"w\") as f:\n json.dump(record_jsons, f)\n\n # save statistics\n execution_stats_dict = data_record_collection.execution_stats.to_json()\n with open(stats_path, \"w\") as f:\n json.dump(execution_stats_dict, f)\n" }, { "path": "abacus-research/helper-scripts/generate-prior-stats-biodex.py", "content": "\"\"\"\nNOTE: this script worked with the tag `abacus-paper-experiments` but is no longer compatible with the main branch.\n\"\"\"\nimport argparse\nimport json\nimport os\nimport time\nfrom functools import partial\n\nimport chromadb\nimport datasets\nfrom chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction\n\n# from ragatouille import RAGPretrainedModel\nimport palimpzest as pz\nfrom palimpzest.constants import Model\n\nbiodex_entry_cols = [\n {\"name\": \"pmid\", \"type\": str, \"desc\": \"The PubMed ID of the medical paper\"},\n {\"name\": \"title\", \"type\": str, \"desc\": \"The title of the medical paper\"},\n {\"name\": \"abstract\", \"type\": str, \"desc\": \"The abstract of the medical paper\"},\n {\"name\": \"fulltext\", \"type\": str, \"desc\": \"The full text of the medical paper, which contains information relevant for creating a drug safety report.\"},\n {\"name\": \"reactions\", \"type\": list[str], \"desc\": \"The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible.\"},\n]\n\nbiodex_reaction_labels_cols = [\n {\"name\": \"reaction_labels\", \"type\": list[str], \"desc\": \"Official terms for medical conditions listed in `reactions`\"},\n]\n\nbiodex_ranked_reactions_labels_cols = [\n {\"name\": \"ranked_reaction_labels\", \"type\": list[str], \"desc\": \"The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs.\"},\n]\n\n\nclass BiodexDataset(pz.IterDataset):\n def __init__(\n self,\n rp_at_k: int = 5,\n num_samples: int = 5,\n split: str = \"test\",\n ):\n super().__init__(id=f\"biodex-{split}\", schema=biodex_entry_cols)\n\n if split == \"test\":\n self.dataset = datasets.load_dataset(\"BioDEX/BioDEX-Reactions\", split=split).to_pandas().to_dict(orient=\"records\")[:num_samples]\n else:\n with open('priors-data/source-idx-to-record-state-cascades.json') as f: # NOTE: unique to cascades run\n self.source_idx_to_record_state = json.load(f)\n self.dataset = [\n self.source_idx_to_record_state[str(idx)]\n for idx in range(5)\n ]\n\n self.rp_at_k = rp_at_k\n self.num_samples = num_samples\n self.split = split\n\n def compute_label(self, entry: dict) -> dict:\n \"\"\"Compute the label for a BioDEX report given its entry in the dataset.\"\"\"\n reactions_lst = [\n reaction.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\")\n for reaction in json.dumps(entry[\"reactions\"]).split(\",\")\n ]\n label_dict = {\n \"reaction_labels\": reactions_lst,\n \"ranked_reaction_labels\": reactions_lst,\n }\n return label_dict\n\n @staticmethod\n def rank_precision_at_k(preds: list | None, targets: list, k: int):\n if preds is None:\n return 0.0\n\n try:\n # lower-case each list\n preds = [pred.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for pred in preds]\n targets = set([target.strip().lower().replace(\"'\", \"\").replace(\"^\", \"\") for target in targets])\n\n # compute rank-precision at k\n rn = len(targets)\n denom = min(k, rn)\n total = 0.0\n for i in range(k):\n total += preds[i] in targets if i < len(preds) else 0.0\n\n return total / denom\n\n except Exception:\n os.makedirs(\"rp@k-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"rp@k-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n @staticmethod\n def term_recall(preds: list | None, targets: list):\n if preds is None:\n return 0.0\n\n try:\n # normalize terms in each list\n pred_terms = set([\n term.strip()\n for pred in preds\n for term in pred.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n target_terms = ([\n term.strip()\n for target in targets\n for term in target.lower().replace(\"'\", \"\").replace(\"^\", \"\").split(\" \")\n ])\n\n # compute term recall and return\n intersect = pred_terms.intersection(target_terms)\n term_recall = len(intersect) / len(target_terms)\n\n return term_recall\n\n except Exception:\n os.makedirs(\"term-recall-eval-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"term-recall-eval-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def __len__(self):\n return len(self.dataset)\n\n def __getitem__(self, idx: int):\n # get entry\n entry = self.dataset[idx]\n\n # get input fields\n pmid = entry[\"pmid\"]\n title = entry[\"title\"]\n abstract = entry[\"abstract\"]\n fulltext = entry[\"fulltext\"]\n reactions = entry[\"reactions\"]\n\n # create item with fields\n item = {\"fields\": {}, \"labels\": {}, \"score_fn\": {}}\n item[\"fields\"][\"pmid\"] = pmid\n item[\"fields\"][\"title\"] = title\n item[\"fields\"][\"abstract\"] = abstract\n item[\"fields\"][\"fulltext\"] = fulltext\n item[\"fields\"][\"reactions\"] = json.dumps(reactions)\n\n if self.split == \"train\":\n # add label info\n item[\"labels\"] = self.compute_label(entry)\n\n # add scoring functions for list fields\n rank_precision_at_k = partial(BiodexDataset.rank_precision_at_k, k=self.rp_at_k)\n item[\"score_fn\"][\"reaction_labels\"] = BiodexDataset.term_recall\n item[\"score_fn\"][\"ranked_reaction_labels\"] = rank_precision_at_k\n\n return item\n\n\nif __name__ == \"__main__\":\n # parse arguments\n parser = argparse.ArgumentParser(description=\"Run a simple demo\")\n parser.add_argument(\"--verbose\", default=False, action=\"store_true\", help=\"Print verbose output\")\n parser.add_argument(\"--progress\", default=True, action=\"store_true\", help=\"Print progress output\")\n args = parser.parse_args()\n\n # create directory for profiling data\n os.makedirs(\"priors-data\", exist_ok=True)\n\n verbose = args.verbose\n progress = args.progress\n seed = 123 # NOTE: unique to cascades run\n execution_strategy = \"parallel\"\n sentinel_execution_strategy = \"all\"\n optimizer_strategy = \"pareto\"\n exp_name = f\"biodex-priors-{optimizer_strategy}-seed{seed}-second-convert-cascades\" # NOTE: unique to cascades run\n\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n # create data source\n dataset = BiodexDataset(\n split=\"test\",\n num_samples=1,\n )\n\n # create validation data source\n train_dataset = BiodexDataset(\n split=\"train\",\n num_samples=5,\n )\n\n # load index [text-embedding-3-small]\n chroma_client = chromadb.PersistentClient(\".chroma-biodex\")\n openai_ef = OpenAIEmbeddingFunction(\n api_key=os.environ[\"OPENAI_API_KEY\"],\n model_name=\"text-embedding-3-small\",\n )\n index = chroma_client.get_collection(\"biodex-reaction-terms\", embedding_function=openai_ef)\n\n def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]:\n # execute query with embeddings\n results = index.query(query, n_results=5)\n\n # get list of result terms with their cosine similarity scores\n final_results = []\n for query_docs, query_distances in zip(results[\"documents\"], results[\"distances\"]):\n for doc, dist in zip(query_docs, query_distances):\n cosine_similarity = 1 - dist\n final_results.append({\"content\": doc, \"similarity\": cosine_similarity})\n\n # sort the results by similarity score\n sorted_results = sorted(final_results, key=lambda result: result[\"similarity\"], reverse=True)\n\n # remove duplicates\n sorted_results_set = set()\n final_sorted_results = []\n for result in sorted_results:\n if result[\"content\"] not in sorted_results_set:\n sorted_results_set.add(result[\"content\"])\n final_sorted_results.append(result[\"content\"])\n\n # return the top-k similar results and generation stats\n return {\"reaction_labels\": final_sorted_results[:k]}\n\n # construct plan\n plan = dataset.sem_topk(\n index=index,\n search_func=search_func,\n search_attr=\"reactions\",\n output_attrs=biodex_reaction_labels_cols,\n )\n plan = plan.sem_add_columns(biodex_ranked_reactions_labels_cols, depends_on=[\"title\", \"abstract\", \"fulltext\", \"reaction_labels\"])\n\n # only use final op quality\n use_final_op_quality = True\n\n # execute pz plan\n config = pz.QueryProcessorConfig(\n optimizer_strategy=optimizer_strategy,\n sentinel_execution_strategy=sentinel_execution_strategy,\n execution_strategy=execution_strategy,\n use_final_op_quality=use_final_op_quality,\n max_workers=64,\n verbose=verbose,\n available_models=[ # NOTE: unique to cascades run\n # Model.GPT_4o,\n Model.GPT_4o_MINI,\n Model.LLAMA3_2_3B,\n Model.LLAMA3_1_8B,\n Model.LLAMA3_3_70B,\n Model.LLAMA3_2_90B_V,\n # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments`\n # Model.DEEPSEEK_V3,\n Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B,\n ],\n allow_bonded_query=True,\n allow_critic=True,\n allow_mixtures=True,\n allow_rag_reduction=True,\n progress=progress,\n k=-1,\n j=-1,\n sample_budget=5*1014 + 5*7,\n seed=seed,\n exp_name=exp_name,\n )\n\n data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=pz.Validator())\n\n print(data_record_collection.to_df())\n data_record_collection.to_df().to_csv(f\"priors-data/{exp_name}-output.csv\", index=False)\n\n # create filepaths for records and stats\n records_path = f\"priors-data/{exp_name}-records.json\"\n stats_path = f\"priors-data/{exp_name}-profiling.json\"\n\n # save record outputs\n record_jsons = []\n for record in data_record_collection:\n record_dict = record.to_dict()\n record_dict = {\n k: v\n for k, v in record_dict.items()\n if k in [\"pmid\", \"reactions\"]\n }\n record_jsons.append(record_dict)\n\n with open(records_path, \"w\") as f:\n json.dump(record_jsons, f)\n\n # save statistics\n execution_stats_dict = data_record_collection.execution_stats.to_json()\n with open(stats_path, \"w\") as f:\n json.dump(execution_stats_dict, f)\n" }, { "path": "abacus-research/helper-scripts/generate-prior-stats-cuad.py", "content": "\"\"\"\nNOTE: this script worked with the tag `abacus-paper-experiments` but is no longer compatible with the main branch.\n\"\"\"\nimport argparse\nimport json\nimport os\nimport string\nfrom functools import partial\n\nimport datasets\nimport numpy as np\nimport pandas as pd\n\nimport palimpzest as pz\nfrom palimpzest.constants import Model\n\ncuad_categories = [\n {\n \"Category\": \"Document Name\",\n \"Description\": \"The name of the contract\",\n \"Answer Format\": \"Contract Name\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Parties\",\n \"Description\": \"The two or more parties who signed the contract\",\n \"Answer Format\": \"Entity or individual names\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Agreement Date\",\n \"Description\": \"The date of the contract\",\n \"Answer Format\": \"Date (mm/dd/yyyy)\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Effective Date\",\n \"Description\": \"The date when the contract is effective\\u00a0\",\n \"Answer Format\": \"Date (mm/dd/yyyy)\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Expiration Date\",\n \"Description\": \"On what date will the contract's initial term expire?\",\n \"Answer Format\": \"Date (mm/dd/yyyy) / Perpetual\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Renewal Term\",\n \"Description\": \"What is the renewal term after the initial term expires? This includes automatic extensions and unilateral extensions with prior notice.\",\n \"Answer Format\": \"[Successive] number of years/months / Perpetual\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Notice Period to Terminate Renewal\",\n \"Description\": \"What is the notice period required to terminate renewal?\",\n \"Answer Format\": \"Number of days/months/year(s)\",\n \"Group\": \"Group: 1\",\n },\n {\n \"Category\": \"Governing Law\",\n \"Description\": \"Which state/country's law governs the interpretation of the contract?\",\n \"Answer Format\": \"Name of a US State / non-US Province, Country\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Most Favored Nation\",\n \"Description\": \"Is there a clause that if a third party gets better terms on the licensing or sale of technology/goods/services described in the contract, the buyer of such technology/goods/services under the contract shall be entitled to those better terms?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Non-Compete\",\n \"Description\": \"Is there a restriction on the ability of a party to compete with the counterparty or operate in a certain geography or business or technology sector?\\u00a0\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 2\",\n },\n {\n \"Category\": \"Exclusivity\",\n \"Description\": \"Is there an exclusive dealing\\u00a0 commitment with the counterparty? This includes a commitment to procure all \\u201crequirements\\u201d from one party of certain technology, goods, or services or a prohibition on licensing or selling technology, goods or services to third parties, or a prohibition on\\u00a0 collaborating or working with other parties), whether during the contract or\\u00a0 after the contract ends (or both).\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 2\",\n },\n {\n \"Category\": \"No-Solicit of Customers\",\n \"Description\": \"Is a party restricted from contracting or soliciting customers or partners of the counterparty, whether during the contract or after the contract ends (or both)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 2\",\n },\n {\n \"Category\": \"Competitive Restriction Exception\",\n \"Description\": \"This category includes the exceptions or carveouts to Non-Compete, Exclusivity and No-Solicit of Customers above.\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 2\",\n },\n {\n \"Category\": \"No-Solicit of Employees\",\n \"Description\": \"Is there a restriction on a party\\u2019s soliciting or hiring employees and/or contractors from the\\u00a0 counterparty, whether during the contract or after the contract ends (or both)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Non-Disparagement\",\n \"Description\": \"Is there a requirement on a party not to disparage the counterparty?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Termination for Convenience\",\n \"Description\": \"Can a party terminate this\\u00a0 contract without cause (solely by giving a notice and allowing a waiting\\u00a0 period to expire)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Rofr/Rofo/Rofn\",\n \"Description\": \"Is there a clause granting one party a right of first refusal, right of first offer or right of first negotiation to purchase, license, market, or distribute equity interest, technology, assets, products or services?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Change of Control\",\n \"Description\": \"Does one party have the right to terminate or is consent or notice required of the counterparty if such party undergoes a change of control, such as a merger, stock sale, transfer of all or substantially all of its assets or business, or assignment by operation of law?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 3\",\n },\n {\n \"Category\": \"Anti-Assignment\",\n \"Description\": \"Is consent or notice required of a party if the contract is assigned to a third party?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 3\",\n },\n {\n \"Category\": \"Revenue/Profit Sharing\",\n \"Description\": \"Is one party required to share revenue or profit with the counterparty for any technology, goods, or\\u00a0services?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Price Restrictions\",\n \"Description\": \"Is there a restriction on the\\u00a0 ability of a party to raise or reduce prices of technology, goods, or\\u00a0 services provided?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Minimum Commitment\",\n \"Description\": \"Is there a minimum order size or minimum amount or units per-time period that one party must buy from the counterparty under the contract?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Volume Restriction\",\n \"Description\": \"Is there a fee increase or consent requirement, etc. if one party\\u2019s use of the product/services exceeds certain threshold?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"IP Ownership Assignment\",\n \"Description\": \"Does intellectual property created\\u00a0 by one party become the property of the counterparty, either per the terms of the contract or upon the occurrence of certain events?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Joint IP Ownership\",\n \"Description\": \"Is there any clause providing for joint or shared ownership of intellectual property between the parties to the contract?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"License Grant\",\n \"Description\": \"Does the contract contain a license granted by one party to its counterparty?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Non-Transferable License\",\n \"Description\": \"Does the contract limit the ability of a party to transfer the license being granted to a third party?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Affiliate License-Licensor\",\n \"Description\": \"Does the contract contain a license grant by affiliates of the licensor or that includes intellectual property of affiliates of the licensor?\\u00a0\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Affiliate License-Licensee\",\n \"Description\": \"Does the contract contain a license grant to a licensee (incl. sublicensor) and the affiliates of such licensee/sublicensor?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Unlimited/All-You-Can-Eat-License\",\n \"Description\": \"Is there a clause granting one party an \\u201centerprise,\\u201d \\u201call you can eat\\u201d or unlimited usage license?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Irrevocable or Perpetual License\",\n \"Description\": \"Does the contract contain a\\u00a0 license grant that is irrevocable or perpetual?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 4\",\n },\n {\n \"Category\": \"Source Code Escrow\",\n \"Description\": \"Is one party required to deposit its source code into escrow with a third party, which can be released to the counterparty upon the occurrence of certain events (bankruptcy,\\u00a0 insolvency, etc.)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Post-Termination Services\",\n \"Description\": \"Is a party subject to obligations after the termination or expiration of a contract, including any post-termination transition, payment, transfer of IP, wind-down, last-buy, or similar commitments?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 5\",\n },\n {\n \"Category\": \"Audit Rights\",\n \"Description\": \"Does a party have the right to\\u00a0 audit the books, records, or physical locations of the counterparty to ensure compliance with the contract?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 5\",\n },\n {\n \"Category\": \"Uncapped Liability\",\n \"Description\": \"Is a party\\u2019s liability uncapped upon the breach of its obligation in the contract? This also includes uncap liability for a particular type of breach such as IP infringement or breach of confidentiality obligation.\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 6\",\n },\n {\n \"Category\": \"Cap on Liability\",\n \"Description\": \"Does the contract include a cap on liability upon the breach of a party\\u2019s obligation? This includes time limitation for the counterparty to bring claims or maximum amount for recovery.\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: 6\",\n },\n {\n \"Category\": \"Liquidated Damages\",\n \"Description\": \"Does the contract contain a clause that would award either party liquidated damages for breach or a fee upon the termination of a contract (termination fee)?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Warranty Duration\",\n \"Description\": \"What is the duration of any\\u00a0 warranty against defects or errors in technology, products, or services\\u00a0 provided under the contract?\",\n \"Answer Format\": \"Number of months or years\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Insurance\",\n \"Description\": \"Is there a requirement for insurance that must be maintained by one party for the benefit of the counterparty?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Covenant Not to Sue\",\n \"Description\": \"Is a party restricted from contesting the validity of the counterparty\\u2019s ownership of intellectual property or otherwise bringing a claim against the counterparty for matters unrelated to the contract?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n {\n \"Category\": \"Third Party Beneficiary\",\n \"Description\": \"Is there a non-contracting party who is a beneficiary to some or all of the clauses in the contract and therefore can enforce its rights against a contracting party?\",\n \"Answer Format\": \"Yes/No\",\n \"Group\": \"Group: -\",\n },\n]\n\nNUM_FIELDS_TO_EXTRACT_PER_CONTRACT = 41\n\n# 0.15 is used in the Doc-ETL paper. It should be 0.5 for the actual benchmark.\nIOU_THRESH = 0.15\n\n\n# Return the Jaccard similarity between two strings\ndef get_jaccard(label, pred):\n remove_tokens = [c for c in string.punctuation if c != \"/\"]\n for token in remove_tokens:\n label = label.replace(token, \"\")\n pred = pred.replace(token, \"\")\n label = label.lower()\n pred = pred.lower()\n label = label.replace(\"/\", \" \")\n pred = pred.replace(\"/\", \" \")\n\n label_words = set(label.split(\" \"))\n pred_words = set(pred.split(\" \"))\n\n intersection = label_words.intersection(pred_words)\n union = label_words.union(pred_words)\n jaccard = len(intersection) / len(union)\n return jaccard\n\n\n# Find the number of true positives, false positives, and false negatives for each entry\n# (one field extracted from each contract) by comparing the labels and predictions.\n# Labels and preds are lists of strings\ndef evaluate_entry(labels, preds, substr_ok):\n tp, fp, fn = 0, 0, 0\n\n # jaccard similarity expects strings\n # TODO: This is a hack, ideally, the return type of the preds should be known\n for idx, pred in enumerate(preds):\n if not isinstance(pred, str):\n print(f\"Expected string, but got {pred}\")\n preds[idx] = str(pred)\n\n # first check if labels is empty\n if len(labels) == 0:\n if len(preds) > 0:\n fp += len(preds) # false positive for each one\n else:\n for ans in labels:\n assert len(ans) > 0\n # check if there is a match\n match_found = False\n for pred in preds:\n if substr_ok:\n is_match = get_jaccard(ans, pred) >= IOU_THRESH or ans in pred\n else:\n is_match = get_jaccard(ans, pred) >= IOU_THRESH\n if is_match:\n match_found = True\n\n if match_found:\n tp += 1\n else:\n fn += 1\n\n # now also get any fps by looping through preds\n for pred in preds:\n # Check if there's a match. if so, don't count (don't want to double count based on the above)\n # but if there's no match, then this is a false positive.\n # (Note: we get the true positives in the above loop instead of this loop so that we don't double count\n # multiple predictions that are matched with the same answer.)\n match_found = False\n for ans in labels:\n assert len(ans) > 0\n if substr_ok:\n is_match = get_jaccard(ans, pred) >= IOU_THRESH or ans in pred\n else:\n is_match = get_jaccard(ans, pred) >= IOU_THRESH\n if is_match:\n match_found = True\n\n if not match_found:\n fp += 1\n\n return tp, fp, fn\n\n\n# TODO(Siva): This is a temporary fix to handle the case where the preds are empty.\ndef handle_empty_preds(preds):\n if preds is None or ( # noqa: SIM114\n isinstance(preds, str) and (preds == \"\" or preds == \" \" or preds == \"null\" or preds == \"None\")\n ):\n return []\n elif isinstance(preds, float) and np.isnan(preds):\n return []\n if not isinstance(preds, (list, np.ndarray)):\n return [preds]\n return preds\n\n\n# Compute the precision and recall for the entire dataset.\n# Each row in the dataframes should correspond to a contract.\n# The columns should be the extracted fields (categories in cuad_categories).\ndef compute_precision_recall(label_df, preds_df):\n tp, fp, fn = 0, 0, 0\n\n label_df = label_df.sort_values(\"contract_id\").reset_index(drop=True)\n preds_df = preds_df.sort_values(\"contract_id\").reset_index(drop=True)\n\n assert label_df.shape == preds_df.shape, (\n f\"Label and prediction dataframes have different shapes, label shape: {label_df.shape} vs preds shape {preds_df.shape}\"\n )\n\n categories = [category[\"Category\"] for category in cuad_categories]\n\n for label_row, pred_row in zip(label_df.iterrows(), preds_df.iterrows()):\n assert label_row[1][\"contract_id\"] == pred_row[1][\"contract_id\"], (\n f\"IDs do not match. label id: {label_row[1]['contract_id']} vs pred id: {pred_row[1]['contract_id']}\"\n )\n for category in categories:\n substr_ok = \"Parties\" in category\n\n labels = label_row[1][category]\n assert isinstance(labels, list)\n\n preds = pred_row[1][category]\n preds = handle_empty_preds(preds)\n\n entry_tp, entry_fp, entry_fn = evaluate_entry(labels, preds, substr_ok)\n tp += entry_tp\n fp += entry_fp\n fn += entry_fn\n\n precision = tp / (tp + fp) if tp + fp > 0 else np.nan\n recall = tp / (tp + fn) if tp + fn > 0 else np.nan\n\n return precision, recall\n\nclass CUADDataset(pz.IterDataset):\n def __init__(self, num_contracts: int = 1, split: str = \"train\", seed: int=42):\n self.num_contracts = num_contracts\n self.split = split\n self.seed = seed\n\n input_cols = [\n {\"name\": \"contract_id\", \"type\": str, \"desc\": \"The id of the the contract to be analyzed\"},\n {\"name\": \"title\", \"type\": str, \"desc\": \"The title of the the contract to be analyzed\"},\n {\"name\": \"contract\", \"type\": str, \"desc\": \"The content of the the contract to be analyzed\"},\n ]\n super().__init__(id=f\"cuad-{split}\", schema=input_cols)\n\n # convert the dataset into a list of dictionaries where each row is for a single contract\n include_labels = split == \"train\"\n dataset = datasets.load_dataset(\"theatticusproject/cuad-qa\")[split]\n self.dataset = self._construct_dataset(dataset, num_contracts, seed, include_labels)\n\n\n def _construct_dataset(self, dataset, num_contracts, seed: int=42, include_labels: bool=False):\n # get the set of unique contract titles; to ensure the order of the contracts is\n # preserved, we use a list rather than using python's set()\n contract_titles = []\n for row in dataset:\n if row[\"title\"] not in contract_titles:\n contract_titles.append(row[\"title\"])\n\n # shuffle the contracts for the given seed\n rng = np.random.default_rng(seed=seed)\n rng.shuffle(contract_titles)\n\n # get the first num_contracts\n contract_titles = contract_titles[:num_contracts]\n\n # construct the dataset one contract at a time\n new_dataset = []\n for title in contract_titles:\n # get the rows for this contract\n contract_rows = [row for row in dataset if row[\"title\"] == title]\n\n # construct the contract; we get the contract_id and contract text from the first row\n contract = {\n \"contract_id\": contract_rows[0][\"id\"],\n \"title\": title,\n \"contract\": contract_rows[0][\"context\"],\n }\n\n # for train / validation data, add the labels\n if include_labels:\n contract = {\"fields\": contract}\n\n # add the labels\n category_names = list(map(lambda category: category[\"Category\"], cuad_categories))\n contract[\"labels\"] = {category: [] for category in category_names}\n contract[\"score_fn\"] = {category: None for category in category_names}\n for row in contract_rows:\n category_name = row[\"id\"].split(\"__\")[-1].split(\"_\")[0].strip()\n category_name = category_name.replace(\" For \", \" for \")\n category_name = category_name.replace(\" Of \", \" of \")\n category_name = category_name.replace(\" On \", \" on \")\n category_name = category_name.replace(\" Or \", \" or \")\n category_name = category_name.replace(\" To \", \" to \")\n category_name = category_name.replace(\"Ip\", \"IP\")\n assert category_name in category_names, f\"Unknown category {category_name}\"\n contract[\"labels\"][category_name].extend(row[\"answers\"][\"text\"])\n\n def score_fn(preds, labels, category_name):\n preds = handle_empty_preds(preds)\n entry_tp, _, entry_fn = evaluate_entry(labels, preds, substr_ok=True) if category_name == \"Parties\" else evaluate_entry(labels, preds, substr_ok=False)\n score = None\n if len(labels) > 0: # noqa: SIM108\n score = entry_tp / (entry_tp + entry_fn)\n else:\n score = 1.0 if len(preds) == 0 else 0.0\n\n return score\n\n contract[\"score_fn\"][category_name] = partial(score_fn, category_name=category_name)\n\n # add the rows to the dataset\n new_dataset.append(contract)\n\n return new_dataset\n\n def __len__(self):\n return self.num_contracts\n\n def __getitem__(self, idx: int):\n return self.dataset[idx]\n\n def get_label_df(self):\n full_dataset = datasets.load_dataset(\"theatticusproject/cuad-qa\")[self.split]\n label_dataset = self._construct_dataset(full_dataset, self.num_contracts, self.seed, True)\n final_label_dataset = []\n for entry in label_dataset:\n row = {}\n row[\"contract_id\"] = entry[\"fields\"][\"contract_id\"]\n row[\"title\"] = entry[\"fields\"][\"title\"]\n row[\"contract\"] = entry[\"fields\"][\"contract\"]\n row = {**row, **entry[\"labels\"]}\n final_label_dataset.append(row)\n\n return pd.DataFrame(final_label_dataset)\n\n\ndef parse_arguments():\n parser = argparse.ArgumentParser(description=\"Run CUAD demo\")\n parser.add_argument(\"--mode\", type=str, help=\"one-convert or separate-converts\", default=\"one-convert\")\n parser.add_argument(\"--test\", type=str, help=\"test time compute active or inactive\", default=\"active\")\n parser.add_argument(\"--verbose\", default=False, action=\"store_true\", help=\"Print verbose output\")\n return parser.parse_args()\n\n\ndef build_cuad_query(dataset, mode):\n assert mode in [\"one-convert\", \"separate-converts\"]\n\n if mode == \"one-convert\":\n cols = []\n for category in cuad_categories:\n desc = (\n f\"Extract the text spans (if they exist) from the contract corresponding to {category['Description']}\"\n )\n cols.append({\"name\": category[\"Category\"], \"type\": list[str], \"desc\": desc})\n\n desc = \"Extract the text spans (if they exist) from the contract.\"\n dataset = dataset.sem_add_columns(cols, desc=desc, depends_on=[\"contract\"])\n elif mode == \"separate-converts\":\n for category in cuad_categories:\n desc = (\n f\"Extract the text spans (if they exist) from the contract corresponding to {category['Description']}\"\n )\n dataset = dataset.sem_add_columns(\n [{\"name\": category[\"Category\"], \"type\": list[str], \"desc\": desc}],\n desc=category[\"Description\"],\n depends_on=[\"contract\"],\n )\n\n return dataset\n\n\ndef main():\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n args = parse_arguments()\n\n # create directory for profiling data\n os.makedirs(\"opt-profiling-data\", exist_ok=True)\n\n # Create a data reader for the CUAD dataset\n dataset = CUADDataset(split=\"test\", num_contracts=1)\n train_dataset = CUADDataset(split=\"train\", num_contracts=5)\n print(\"Created data reader\")\n\n # Build and run the CUAD query\n query = build_cuad_query(dataset, args.mode)\n print(\"Built query; Starting query execution\")\n\n execution_strategy = \"parallel\"\n sentinel_execution_strategy = \"all\"\n optimizer_strategy = \"pareto\"\n seed = 0\n exp_name = f\"cuad-priors-{optimizer_strategy}-seed{seed}\"\n config = pz.QueryProcessorConfig(\n verbose=False,\n optimizer_strategy=optimizer_strategy,\n sentinel_execution_strategy=sentinel_execution_strategy,\n execution_strategy=execution_strategy,\n max_workers=64,\n available_models=[\n Model.GPT_4o,\n Model.GPT_4o_MINI,\n # Model.LLAMA3_2_3B,\n Model.LLAMA3_1_8B,\n Model.LLAMA3_3_70B,\n # Model.LLAMA3_2_90B_V,\n # Model.MIXTRAL, # NOTE: only available in tag `abacus-paper-experiments`\n # Model.DEEPSEEK_V3,\n Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B,\n ],\n allow_bonded_query=True,\n allow_critic=True,\n allow_mixtures=True,\n allow_rag_reduction=True,\n progress=True,\n k=-1,\n j=-1,\n sample_budget=1014*5,\n seed=seed,\n exp_name=exp_name,\n )\n data_record_collection = query.optimize_and_run(config=config, train_dataset=train_dataset, validator=pz.Validator())\n print(\"Query execution completed\")\n\n # save statistics\n execution_stats_dict = data_record_collection.execution_stats.to_json()\n with open(f\"priors-data/{exp_name}-stats.json\", \"w\") as f:\n json.dump(execution_stats_dict, f)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "abacus-research/helper-scripts/mmqa-baseline.py", "content": "import argparse\nimport json\nimport os\nimport string\nimport time\n\nimport numpy as np\nfrom openai import OpenAI\n\nfrom palimpzest.constants import Cardinality, Model\nfrom palimpzest.query.generators.generators import get_json_from_answer\n\n\ndef f1(preds: list | None, targets: list):\n if preds is None or len(targets) == 0:\n return 0.0\n\n tp, fp, fn = 0, 0, 0\n try:\n # compute recall of retrieved ids and return\n preds = [str(pred).lower() for pred in preds]\n targets = [str(target).lower() for target in targets]\n\n remove_tokens = [c for c in string.punctuation if c != \"/\"]\n for token in remove_tokens:\n preds = [pred.replace(token, \"\") for pred in preds]\n targets = [target.replace(token, \"\") for target in targets]\n\n for pred in preds:\n if pred in targets:\n tp += 1\n else:\n fp += 1\n for target in targets:\n if target not in preds:\n fn += 1\n\n # compute overall f1 score and return\n recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n\n return f1\n\n except Exception:\n os.makedirs(\"mmqa-recall-eval-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"mmqa-recall-eval-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n\nif __name__ == \"__main__\":\n # parse arguments\n parser = argparse.ArgumentParser(description=\"Run a simple demo\")\n parser.add_argument(\n \"--seed\",\n default=42,\n type=int,\n help=\"Seed used to initialize RNG for MAB sampling algorithm\",\n )\n args = parser.parse_args()\n\n # create directory for profiling data\n os.makedirs(\"opt-profiling-data\", exist_ok=True)\n seed = args.seed\n print(f\"Running with seed: {seed}\")\n\n # start time for processing\n start_time = time.time()\n\n # read the appropriate dataset\n dataset = []\n with open(\"data/MMQA_dev.jsonl\") as f:\n for line in f:\n dict_line = json.loads(line)\n if \"image\" in dict_line[\"metadata\"][\"modalities\"] and len(dict_line[\"metadata\"][\"modalities\"]) > 1:\n dataset.append(dict_line)\n\n # shuffle the questions for the given seed\n rng = np.random.default_rng(seed=seed)\n rng.shuffle(dataset)\n\n # trim to number of samples\n dataset = dataset[:100]\n\n # construct the prompt\n prompt = \"\"\"You are an intelligent AI assistant designed to answer questions. Please answer the following question to the best of your ability based on your prior knowledge.\n Return your answer as a JSON list of strings.\n Do not include any additional context or an explanation in your answer, simply list the entities asked for by the question:\n\nQUESTION: {question}\n\nANSWER: \n\"\"\"\n\n # iterate over the dataset and generate answers\n model_name = \"gpt-4o-mini-2024-07-18\"\n preds, total_cost = [], 0.0\n for idx, entry in enumerate(dataset):\n print(f\"Processing entry {idx}\")\n formatted_prompt = prompt.format(question=entry[\"question\"])\n client = OpenAI()\n payload = {\n \"model\": model_name,\n \"temperature\": 0.0,\n \"messages\": [{\"role\": \"user\", \"content\": formatted_prompt}],\n }\n completion = client.chat.completions.create(**payload)\n\n # compute total cost\n model = Model(model_name)\n usd_per_input_token = model.get_usd_per_input_token()\n usd_per_output_token = model.get_usd_per_output_token()\n input_tokens = completion.usage.prompt_tokens\n output_tokens = completion.usage.completion_tokens\n total_cost += input_tokens * usd_per_input_token + output_tokens * usd_per_output_token\n\n # extract answer\n completion_text = completion.choices[0].message.content\n try:\n answer = get_json_from_answer(completion_text, Model.GPT_4o_MINI, Cardinality.ONE_TO_MANY)\n except Exception:\n answer = [completion_text]\n preds.append(answer)\n\n # get total time\n total_time = time.time() - start_time\n\n # score the output\n scores = []\n for pred, entry in zip(preds, dataset):\n answers = entry[\"answers\"]\n answer = [ans[\"answer\"] for ans in answers]\n f1_score = f1(pred, answer)\n scores.append(f1_score)\n\n # create final stats dict\n stats = {}\n stats[\"total_time\"] = total_time\n stats[\"total_cost\"] = total_cost\n stats[\"f1\"] = np.mean(scores)\n with open(f'opt-profiling-data/mmqa-baseline-seed-{seed}-stats.json', 'w') as f:\n json.dump(stats, f)\n print(stats)\n" }, { "path": "abacus-research/helper-scripts/mmqa-gen-image-index.py", "content": "import json\nimport os\n\nimport chromadb\nimport chromadb.utils.embedding_functions as embedding_functions\nimport numpy as np\nfrom PIL import Image\nfrom sentence_transformers import SentenceTransformer\nfrom tqdm import tqdm\n\nCORRUPTED_IMAGE_IDS = [\n \"17ae0616ac745e70781203267f3a382d\",\n \"bf201cbbd058ef51aef89b1be4158c2a\",\n \"ef457a7b3ab437cd78ab9f82dc083048\",\n \"225c3db49d60b5ef30ed0bfc649ebf78\",\n \"b413cc1dc4969dcbe4cb6a55c0f2e359\",\n \"e81b2acfd792b171389c8f47a0e14504\",\n]\n\n# NOTE: this script is meant to be run from the root of the repository\nif __name__ == \"__main__\":\n # load CLIP model\n model = SentenceTransformer(\"clip-ViT-B-32\")\n\n # load image metadata\n image_filepaths, image_ids = [], []\n with open(\"data/MMQA_images.jsonl\") as f:\n possible_endings = {'.JPG', '.png', '.jpeg', '.jpg', '.tif', '.JPEG', '.tiff', '.PNG', '.Jpg', '.gif'}\n for line in f:\n dict_line = json.loads(line)\n image_id = dict_line[\"id\"]\n\n # skip corrupted images:\n if image_id in CORRUPTED_IMAGE_IDS:\n continue\n\n # add image to image_ids\n image_ids.append(image_id)\n\n # find the correct image file\n for ending in possible_endings:\n if os.path.exists(f\"/ssd1/mdrusso/mmqa-images/{image_id}{ending}\"):\n image_id += ending\n break\n\n image_filepaths.append(f\"/ssd1/mdrusso/mmqa-images/{image_id}\")\n\n # create directory for embeddings\n os.makedirs(\"testdata/mmqa-image-embeddings/\", exist_ok=True)\n\n # generate embeddings in batches of 128 at a time\n indices = np.linspace(0, len(image_filepaths), len(image_filepaths)//128, dtype=int)\n total_embeds = len(indices)\n print(f\"Generating {total_embeds} batches of embeddings...\")\n gen_indices = []\n for iter_idx, start_idx in tqdm(enumerate(indices), total=total_embeds):\n # check if embedding needs to be computed\n end_idx = indices[iter_idx + 1] if iter_idx + 1 < len(indices) else None\n filename = f\"testdata/mmqa-image-embeddings/{start_idx}_{end_idx}.npy\"\n if end_idx is not None and not os.path.exists(filename):\n # generate embeddings\n batch_fps = image_filepaths[start_idx:end_idx]\n batch_images = [Image.open(fp) for fp in batch_fps]\n embeddings = model.encode(batch_images)\n \n # save embeddings to disk\n with open(filename, \"wb\") as f:\n np.save(f, embeddings)\n\n gen_indices.append((start_idx, end_idx))\n print(\"Done generating embeddings.\")\n\n # initialize chroma client\n chroma_client = chromadb.PersistentClient(\".chroma-mmqa\")\n\n # initialize embedding function\n sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(\n model_name=\"clip-ViT-B-32\"\n )\n\n # create a collection\n collection = chroma_client.get_or_create_collection(\n name=\"mmqa-images\",\n embedding_function=sentence_transformer_ef,\n metadata={\"hnsw:space\": \"cosine\"},\n )\n\n # insert documents in batches\n total_inserts = len(gen_indices)\n print(f\"Inserting {total_inserts} batches into the collection...\")\n for start_idx, end_idx in tqdm(gen_indices, total=total_inserts):\n embeddings = np.load(f\"testdata/mmqa-image-embeddings/{start_idx}_{end_idx}.npy\")\n collection.add(\n documents=[os.path.basename(fp) for fp in image_filepaths[start_idx:end_idx]],\n embeddings=embeddings.tolist(),\n ids=image_ids[start_idx:end_idx],\n )\n" }, { "path": "abacus-research/helper-scripts/mmqa-gen-image-title-index.py", "content": "import json\nimport os\nimport time\n\nimport chromadb\nimport chromadb.utils.embedding_functions as embedding_functions\nimport numpy as np\nfrom openai import OpenAI\nfrom tqdm import tqdm\n\n# NOTE: this script is meant to be run from the root of the repository\nif __name__ == \"__main__\":\n # initialize openai client\n openai_client = OpenAI()\n\n # load image metadata\n image_title_set = set()\n image_titles, image_ids = [], []\n with open(\"data/MMQA_images.jsonl\") as f:\n for line in f:\n dict_line = json.loads(line)\n image_title = dict_line[\"title\"]\n if image_title == \"\":\n image_title = dict_line[\"url\"]\n\n if image_title not in image_title_set:\n image_titles.append(image_title)\n image_title_set.add(image_title)\n else:\n idx = 1\n while f\"{image_title} ({idx})\" in image_title_set:\n idx += 1\n image_title = f\"{image_title} ({idx})\"\n image_titles.append(image_title)\n image_title_set.add(image_title)\n\n image_ids.append(dict_line[\"id\"])\n\n # create directory for embeddings\n os.makedirs(\"testdata/mmqa-image-title-embeddings/\", exist_ok=True)\n\n # generate embeddings in batches of 1000 at a time\n indices = np.linspace(0, len(image_titles), len(image_titles)//1000, dtype=int)\n total_embeds = len(indices)\n print(f\"Generating {total_embeds} batches of embeddings...\")\n gen_indices = []\n for iter_idx, start_idx in tqdm(enumerate(indices), total=total_embeds):\n # check if embedding needs to be computed\n end_idx = indices[iter_idx + 1] if iter_idx + 1 < len(indices) else None\n filename = f\"testdata/mmqa-image-title-embeddings/{start_idx}_{end_idx}.npy\"\n if end_idx is not None and not os.path.exists(filename):\n # generate embeddings\n batch = image_titles[start_idx:end_idx]\n resp = openai_client.embeddings.create(input=batch, model=\"text-embedding-3-small\")\n embeddings = [item.embedding for item in resp.data]\n\n # save embeddings to disk\n with open(filename, \"wb\") as f:\n np.save(f, np.array(embeddings))\n\n gen_indices.append((start_idx, end_idx))\n time.sleep(1)\n print(\"Done generating embeddings.\")\n\n # initialize chroma client\n chroma_client = chromadb.PersistentClient(\".chroma-mmqa\")\n\n # initialize embedding function\n openai_ef = embedding_functions.OpenAIEmbeddingFunction(\n api_key=os.environ[\"OPENAI_API_KEY\"],\n model_name=\"text-embedding-3-small\"\n )\n\n # create a collection\n collection = chroma_client.get_or_create_collection(\n name=\"mmqa-image-titles\",\n embedding_function=openai_ef,\n metadata={\"hnsw:space\": \"cosine\"},\n )\n\n # insert documents in batches\n total_inserts = len(gen_indices)\n print(f\"Inserting {total_inserts} batches into the collection...\")\n for start_idx, end_idx in tqdm(gen_indices, total=total_inserts):\n embeddings = np.load(f\"testdata/mmqa-image-title-embeddings/{start_idx}_{end_idx}.npy\")\n collection.add(\n documents=image_titles[start_idx:end_idx],\n embeddings=embeddings.tolist(),\n ids=image_ids[start_idx:end_idx],\n )\n" }, { "path": "abacus-research/helper-scripts/mmqa-gen-table-index.py", "content": "import json\nimport os\nimport time\n\nimport chromadb\nimport chromadb.utils.embedding_functions as embedding_functions\nimport numpy as np\nfrom openai import OpenAI\nfrom tqdm import tqdm\n\n# NOTE: this script is meant to be run from the root of the repository\nif __name__ == \"__main__\":\n # initialize openai client\n openai_client = OpenAI()\n\n # load table texts\n table_texts, table_ids = [], []\n with open(\"data/MMQA_tables.jsonl\") as f:\n for line in f:\n dict_line = json.loads(line)\n \n # get page title and table name\n page_title = dict_line[\"title\"]\n table_name = dict_line[\"table\"][\"table_name\"]\n\n # get table column names and empty column indices\n table_header = dict_line[\"table\"][\"header\"]\n column_names = [col[\"column_name\"] for col in table_header if col[\"column_name\"] != \"\"]\n empty_col_indices = set([idx for idx, col in enumerate(table_header) if col[\"column_name\"] == \"\"])\n\n # create string for table data\n text = f\"{page_title}: {table_name}\\n\\n{','.join(column_names)}\\n\"\n\n # parse table row data\n table_rows = dict_line[\"table\"][\"table_rows\"]\n for row in table_rows:\n row_data = []\n for idx, cell in enumerate(row):\n if idx in empty_col_indices:\n continue\n row_data.append(cell[\"text\"])\n\n text += \",\".join(row_data) + \"\\n\"\n\n table_texts.append(text)\n table_ids.append(dict_line[\"id\"])\n\n # create directory for embeddings\n os.makedirs(\"testdata/mmqa-table-embeddings/\", exist_ok=True)\n\n # generate embeddings in batches of 1000 at a time\n indices = np.linspace(0, len(table_texts), len(table_texts)//1000, dtype=int)\n total_embeds = len(indices)\n print(f\"Generating {total_embeds} batches of embeddings...\")\n gen_indices = []\n for iter_idx, start_idx in tqdm(enumerate(indices), total=total_embeds):\n # check if embedding needs to be computed\n end_idx = indices[iter_idx + 1] if iter_idx + 1 < len(indices) else None\n filename = f\"testdata/mmqa-table-embeddings/{start_idx}_{end_idx}.npy\"\n if end_idx is not None and not os.path.exists(filename):\n # generate embeddings\n batch = table_texts[start_idx:end_idx]\n resp = openai_client.embeddings.create(input=batch, model=\"text-embedding-3-small\")\n embeddings = [item.embedding for item in resp.data]\n\n # save embeddings to disk\n with open(filename, \"wb\") as f:\n np.save(f, np.array(embeddings))\n\n gen_indices.append((start_idx, end_idx))\n time.sleep(1)\n print(\"Done generating embeddings.\")\n\n # initialize chroma client\n chroma_client = chromadb.PersistentClient(\".chroma-mmqa\")\n\n # initialize embedding function\n openai_ef = embedding_functions.OpenAIEmbeddingFunction(\n api_key=os.environ[\"OPENAI_API_KEY\"],\n model_name=\"text-embedding-3-small\"\n )\n\n # create a collection\n collection = chroma_client.get_or_create_collection(\n name=\"mmqa-tables\",\n embedding_function=openai_ef,\n metadata={\"hnsw:space\": \"cosine\"},\n )\n\n # insert documents in batches\n total_inserts = len(gen_indices)\n print(f\"Inserting {total_inserts} batches into the collection...\")\n for start_idx, end_idx in tqdm(gen_indices, total=total_inserts):\n embeddings = np.load(f\"testdata/mmqa-table-embeddings/{start_idx}_{end_idx}.npy\")\n collection.add(\n documents=table_texts[start_idx:end_idx],\n embeddings=embeddings.tolist(),\n ids=table_ids[start_idx:end_idx],\n )\n" }, { "path": "abacus-research/helper-scripts/mmqa-gen-text-index.py", "content": "import json\nimport os\nimport time\n\nimport chromadb\nimport chromadb.utils.embedding_functions as embedding_functions\nimport numpy as np\nfrom openai import OpenAI\nfrom tqdm import tqdm\n\n# NOTE: this script is meant to be run from the root of the repository\nif __name__ == \"__main__\":\n # initialize openai client\n openai_client = OpenAI()\n\n # load texts\n texts, text_ids = [], []\n with open(\"data/MMQA_texts.jsonl\") as f:\n for line in f:\n dict_line = json.loads(line)\n title = dict_line[\"title\"]\n text = dict_line[\"text\"]\n texts.append(f\"{title}: {text}\")\n text_ids.append(dict_line[\"id\"])\n\n # create directory for embeddings\n os.makedirs(\"testdata/mmqa-text-embeddings/\", exist_ok=True)\n\n # generate embeddings in batches of 1000 at a time\n indices = np.linspace(0, len(texts), len(texts)//1000, dtype=int)\n total_embeds = len(indices)\n print(f\"Generating {total_embeds} batches of embeddings...\")\n gen_indices = []\n for iter_idx, start_idx in tqdm(enumerate(indices), total=total_embeds):\n # check if embedding needs to be computed\n end_idx = indices[iter_idx + 1] if iter_idx + 1 < len(indices) else None\n filename = f\"testdata/mmqa-text-embeddings/{start_idx}_{end_idx}.npy\"\n if end_idx is not None and not os.path.exists(filename):\n # generate embeddings\n batch = texts[start_idx:end_idx]\n resp = openai_client.embeddings.create(input=batch, model=\"text-embedding-3-small\")\n embeddings = [item.embedding for item in resp.data]\n\n # save embeddings to disk\n with open(filename, \"wb\") as f:\n np.save(f, np.array(embeddings))\n\n gen_indices.append((start_idx, end_idx))\n time.sleep(1)\n print(\"Done generating embeddings.\")\n\n # initialize chroma client\n chroma_client = chromadb.PersistentClient(\".chroma-mmqa\")\n\n # initialize embedding function\n openai_ef = embedding_functions.OpenAIEmbeddingFunction(\n api_key=os.environ[\"OPENAI_API_KEY\"],\n model_name=\"text-embedding-3-small\"\n )\n\n # create a collection\n collection = chroma_client.get_or_create_collection(\n name=\"mmqa-texts\",\n embedding_function=openai_ef,\n metadata={\"hnsw:space\": \"cosine\"},\n )\n\n # insert documents in batches\n total_inserts = len(gen_indices)\n print(f\"Inserting {total_inserts} batches into the collection...\")\n for start_idx, end_idx in tqdm(gen_indices, total=total_inserts):\n embeddings = np.load(f\"testdata/mmqa-text-embeddings/{start_idx}_{end_idx}.npy\")\n collection.add(\n documents=texts[start_idx:end_idx],\n embeddings=embeddings.tolist(),\n ids=text_ids[start_idx:end_idx],\n )\n" }, { "path": "abacus-research/mmqa-complex-demo.py", "content": "import argparse\nimport base64\nimport json\nimport os\nimport string\nimport time\n\nimport numpy as np\nimport regex as re\n\nimport palimpzest as pz\nfrom palimpzest.constants import Model\nfrom palimpzest.core.lib.schemas import ImageBase64\n\nCORRUPTED_IMAGE_IDS = [\n \"17ae0616ac745e70781203267f3a382d\",\n \"bf201cbbd058ef51aef89b1be4158c2a\",\n \"ef457a7b3ab437cd78ab9f82dc083048\",\n \"225c3db49d60b5ef30ed0bfc649ebf78\",\n \"b413cc1dc4969dcbe4cb6a55c0f2e359\",\n \"e81b2acfd792b171389c8f47a0e14504\",\n]\n\nmmqa_entry_cols = [\n {\"name\": \"qid\", \"type\": str, \"desc\": \"The id of the MMQA question\"},\n {\"name\": \"question\", \"type\": str, \"desc\": \"The question which needs to be answered\"},\n]\nmmqa_text_search_cols = [\n {\"name\": \"text_search_string\", \"type\": str, \"desc\": \"A string used to search for relevant text snippets.\"},\n]\nmmqa_table_search_cols = [\n {\"name\": \"table_search_string\", \"type\": str, \"desc\": \"A string used to search for relevant tables.\"},\n]\nmmqa_image_search_cols = [\n {\"name\": \"image_search_string\", \"type\": str, \"desc\": \"A string used to search for relevant images.\"},\n]\n\nmmqa_text_cols = [\n {\"name\": \"text_id\", \"type\": str, \"desc\": \"The id for the given text snippet.\"},\n {\"name\": \"text\", \"type\": str, \"desc\": \"A text snippet which may or may not support the question.\"},\n]\n\nmmqa_table_cols = [\n {\"name\": \"table_id\", \"type\": str, \"desc\": \"The id for the given table.\"},\n {\"name\": \"table\", \"type\": str, \"desc\": \"A table which may or may not support the question.\"},\n]\n\nmmqa_image_cols = [\n {\"name\": \"image_id\", \"type\": str, \"desc\": \"The id for the given image.\"},\n {\"name\": \"image\", \"type\": ImageBase64, \"desc\": \"An image which may or may not support the question.\"},\n]\n\nmmqa_answer_cols = [\n {\"name\": \"answers\", \"type\": list[str], \"desc\": \"The answer(s) to the question. Answer the question using the relevant information from gathered image(s), text(s), and table(s). Return your answer as a JSON list of strings. Do not include any additional context or an explanation in your answer, simply list the entities asked for by the question\"},\n]\n\ndef get_json_from_answer(answer: str):\n \"\"\"\n This function parses an LLM response which is supposed to output a JSON object\n and optimistically searches for the substring containing the JSON object.\n \"\"\"\n # split off context / excess, which models sometimes output after answer\n answer = answer.split(\"Context:\")[0]\n answer = answer.split(\"# this is the answer\")[0]\n # trim the answer to only include the JSON array\n if not answer.strip().startswith(\"[\"):\n # Find the start index of the actual JSON string assuming the prefix is followed by the JSON array\n start_index = answer.find(\"[\")\n if start_index != -1:\n # Remove the prefix and any leading characters before the JSON starts\n answer = answer[start_index:]\n if not answer.strip().endswith(\"]\"):\n # Find the end index of the actual JSON string\n # assuming the suffix is preceded by the JSON object/array\n end_index = answer.rfind(\"]\")\n if end_index != -1:\n # Remove the suffix and any trailing characters after the JSON ends\n answer = answer[: end_index + 1]\n # Handle weird escaped values. I am not sure why the model\n # is returning these, but the JSON parser can't take them\n answer = answer.replace(r\"\\_\", \"_\")\n answer = answer.replace(\"\\\\n\", \"\\n\")\n # Remove https and http prefixes to not conflict with comment detection\n # Handle comments in the JSON response. Use regex from // until end of line\n answer = re.sub(r\"(? dict:\n \"\"\"Compute the label for a MMQA question given its entry in the dataset.\"\"\"\n qid_to_labels = {}\n for entry in self.dataset:\n # get the answers\n answers = [answer[\"answer\"] for answer in entry[\"answers\"]]\n supporting_text_ids = [context[\"doc_id\"] for context in entry[\"supporting_context\"] if context[\"doc_part\"] == \"text\"]\n supporting_table_ids = [context[\"doc_id\"] for context in entry[\"supporting_context\"] if context[\"doc_part\"] == \"table\"]\n supporting_image_ids = [context[\"doc_id\"] for context in entry[\"supporting_context\"] if context[\"doc_part\"] == \"image\"]\n\n label_dict = {\n \"answers\": answers,\n \"supporting_text_ids\": supporting_text_ids,\n \"supporting_table_ids\": supporting_table_ids,\n \"supporting_image_ids\": supporting_image_ids,\n }\n qid_to_labels[entry[\"qid\"]] = label_dict\n\n return qid_to_labels\n\n def recall(self, preds: list | None, targets: list):\n if preds is None or len(targets) == 0:\n return 0.0\n\n tp, fn = 0, 0\n try:\n # compute recall of retrieved ids and return\n preds = [str(pred).lower() for pred in preds]\n targets = [str(target).lower() for target in targets]\n remove_tokens = [c for c in string.punctuation if c != \"/\"]\n for token in remove_tokens:\n preds = [pred.replace(token, \"\") for pred in preds]\n targets = [target.replace(token, \"\") for target in targets]\n\n\n for target in targets:\n if target in preds:\n tp += 1\n else:\n fn += 1\n\n return tp / (tp + fn)\n\n except Exception:\n os.makedirs(\"mmqa-recall-eval-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"mmqa-recall-eval-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def f1(self, preds: list | None, targets: list):\n if preds is None or len(targets) == 0:\n return 0.0\n\n tp, fp, fn = 0, 0, 0\n try:\n # compute recall of retrieved ids and return\n preds = [str(pred).lower() for pred in preds]\n targets = [str(target).lower() for target in targets]\n\n remove_tokens = [c for c in string.punctuation if c != \"/\"]\n for token in remove_tokens:\n preds = [pred.replace(token, \"\") for pred in preds]\n targets = [target.replace(token, \"\") for target in targets]\n\n for pred in preds:\n if pred in targets:\n tp += 1\n else:\n fp += 1\n for target in targets:\n if target not in preds:\n fn += 1\n\n # compute overall f1 score and return\n recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n\n return f1\n\n except Exception:\n os.makedirs(\"mmqa-recall-eval-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"mmqa-recall-eval-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n if \"answers\" not in fields:\n return None\n preds = output.get(\"answers\")\n targets = self.qid_to_labels[str(input_record[\"qid\"])][\"answers\"]\n return self.f1(preds, targets)\n\n def join_score_fn(self, condition: str, left_input_record: dict, right_input_record: dict, output: bool) -> float | None:\n if condition == \"The text snippet is relevant to the question based on the text search string.\":\n pred = right_input_record[\"text_id\"]\n targets = self.qid_to_labels[left_input_record[\"qid\"]][\"supporting_text_ids\"]\n return pred in targets and output or pred not in targets and not output\n elif condition == \"The table is relevant to the question based on the table search string.\":\n pred = right_input_record[\"table_id\"]\n targets = self.qid_to_labels[left_input_record[\"qid\"]][\"supporting_table_ids\"]\n return pred in targets and output or pred not in targets and not output\n elif condition == \"The image is relevant to the question based on the image search string.\":\n pred = right_input_record[\"image_id\"]\n targets = self.qid_to_labels[left_input_record[\"qid\"]][\"supporting_image_ids\"]\n return pred in targets and output or pred not in targets and not output\n else:\n raise NotImplementedError(f\"Validator.join_score_fn not implemented for condition {condition}.\")\n\n\nclass MMQAQuestionDataset(pz.IterDataset):\n def __init__(self, dataset: list[dict]):\n super().__init__(id=\"mmqa-questions\", schema=mmqa_entry_cols)\n self.dataset = [{\"qid\": entry[\"qid\"], \"question\": entry[\"question\"]} for entry in dataset]\n\n def __len__(self):\n return len(self.dataset)\n\n def __getitem__(self, idx: int):\n return self.dataset[idx]\n\n\nclass MMQATextDataset(pz.IterDataset):\n def __init__(self, dataset: list[dict]):\n super().__init__(id=\"mmqa-texts\", schema=mmqa_text_cols)\n\n # construct mapping from text id to text\n text_id_to_text = {}\n with open(\"data/MMQA_texts.jsonl\") as f:\n for line in f:\n dict_line = json.loads(line)\n text_id_to_text[dict_line[\"id\"]] = f\"{dict_line['title']}: {dict_line['text']}\"\n\n # construct dataset\n self.dataset = []\n for entry in dataset:\n for context in entry[\"supporting_context\"]:\n if context[\"doc_part\"] == \"text\":\n text_id = context[\"doc_id\"]\n text = text_id_to_text[text_id]\n self.dataset.append({\"text_id\": text_id, \"text\": text})\n\n def __len__(self):\n return len(self.dataset)\n\n def __getitem__(self, idx: int):\n return self.dataset[idx]\n\n\nclass MMQATableDataset(pz.IterDataset):\n def __init__(self, dataset: list[dict]):\n super().__init__(id=\"mmqa-tables\", schema=mmqa_table_cols)\n\n # construct mapping from table id to table string\n table_id_to_table = {}\n with open(\"data/MMQA_tables.jsonl\") as f:\n for line in f:\n dict_line = json.loads(line)\n\n # get page title and table name\n page_title = dict_line[\"title\"]\n table_name = dict_line[\"table\"][\"table_name\"]\n\n # get table column names and empty column indices\n table_header = dict_line[\"table\"][\"header\"]\n column_names = [col[\"column_name\"] for col in table_header if col[\"column_name\"] != \"\"]\n empty_col_indices = set([idx for idx, col in enumerate(table_header) if col[\"column_name\"] == \"\"])\n\n # create string for table data\n text = f\"{page_title}: {table_name}\\n\\n{','.join(column_names)}\\n\"\n\n # parse table row data\n table_rows = dict_line[\"table\"][\"table_rows\"]\n for row in table_rows:\n row_data = []\n for idx, cell in enumerate(row):\n if idx in empty_col_indices:\n continue\n row_data.append(cell[\"text\"])\n\n text += \",\".join(row_data) + \"\\n\"\n\n table_id_to_table[dict_line[\"id\"]] = text\n\n # construct dataset\n self.dataset = []\n for entry in dataset:\n for context in entry[\"supporting_context\"]:\n if context[\"doc_part\"] == \"table\":\n table_id = context[\"doc_id\"]\n table = table_id_to_table[table_id]\n self.dataset.append({\"table_id\": table_id, \"table\": table})\n\n def __len__(self):\n return len(self.dataset)\n\n def __getitem__(self, idx: int):\n return self.dataset[idx]\n\n\nclass MMQAImageDataset(pz.IterDataset):\n def __init__(self, dataset: list[dict]):\n super().__init__(id=\"mmqa-images\", schema=mmqa_image_cols)\n\n # construct mapping from image id to image base64 object\n image_id_to_image = {}\n with open(\"data/MMQA_images.jsonl\") as f:\n possible_endings = {'.JPG', '.png', '.jpeg', '.jpg', '.tif', '.JPEG', '.tiff', '.PNG', '.Jpg', '.gif'}\n for line in f:\n dict_line = json.loads(line)\n image_id = dict_line[\"id\"]\n\n # skip corrupted images:\n if image_id in CORRUPTED_IMAGE_IDS:\n continue\n\n # find the correct image file\n image_filepath = None\n for ending in possible_endings:\n filepath = f\"data/final_dataset_images/{image_id}{ending}\"\n if os.path.exists(filepath):\n image_filepath = filepath\n break\n\n # read the image file and convert to base64\n with open(image_filepath, \"rb\") as f:\n contents = base64.b64encode(f.read()).decode(\"utf-8\")\n image_id_to_image[image_id] = contents\n\n # construct dataset\n self.dataset = []\n for entry in dataset:\n for context in entry[\"supporting_context\"]:\n if context[\"doc_part\"] == \"image\":\n image_id = context[\"doc_id\"]\n image = image_id_to_image[image_id]\n self.dataset.append({\"image_id\": image_id, \"image\": image})\n\n def __len__(self):\n return len(self.dataset)\n\n def __getitem__(self, idx: int):\n return self.dataset[idx]\n\n\ndef get_dataset(split: str, shuffle: bool, seed: int, num_samples: int | None) -> list[str]:\n dataset = []\n with open(f\"data/MMQA_{split}.jsonl\") as f:\n for line in f:\n dict_line = json.loads(line)\n if \"image\" in dict_line[\"metadata\"][\"modalities\"] and len(dict_line[\"metadata\"][\"modalities\"]) > 1:\n dataset.append(dict_line)\n\n # shuffle the questions for the given seed\n if shuffle:\n rng = np.random.default_rng(seed=seed)\n rng.shuffle(dataset)\n \n return dataset if num_samples is None else dataset[:num_samples]\n\n\ndef compute_f1(final_df, answers_df):\n merged_df = final_df.merge(answers_df, on=\"qid\", how=\"left\")\n tp, fp, fn = 0, 0, 0\n for _, row in merged_df.iterrows():\n targets = [str(target).lower() for target in row[\"gt_answers\"]]\n preds = row[\"answers\"]\n if isinstance(preds, str):\n try:\n # convert single quotes to double quotes before parsing for JSON\n preds = preds.replace(\"'\", '\"')\n # try parsing preds as JSON list and cast everything to str to match targets\n preds = get_json_from_answer(preds)\n preds = [str(pred).lower() for pred in preds]\n except Exception:\n # if that fails, give it a shot as a singleton answer that the LLM failed to wrap in a list\n preds = [preds.lower()]\n remove_tokens = [c for c in string.punctuation if c != \"/\"]\n for token in remove_tokens:\n preds = [pred.replace(token, \"\") for pred in preds]\n targets = [target.replace(token, \"\") for target in targets]\n elif isinstance(preds, list):\n preds = [str(pred).lower() for pred in preds]\n remove_tokens = [c for c in string.punctuation if c != \"/\"]\n for token in remove_tokens:\n preds = [pred.replace(token, \"\") for pred in preds]\n targets = [target.replace(token, \"\") for target in targets]\n else:\n preds = []\n for pred in preds:\n if pred in targets:\n tp += 1\n else:\n fp += 1\n for target in targets:\n if target not in preds:\n fn += 1\n # compute overall f1 score and return\n recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n return f1\n\n\nif __name__ == \"__main__\":\n # parse arguments\n parser = argparse.ArgumentParser(description=\"Run a simple demo\")\n parser.add_argument(\"--verbose\", default=False, action=\"store_true\", help=\"Print verbose output\")\n parser.add_argument(\"--progress\", default=False, action=\"store_true\", help=\"Print progress output\")\n parser.add_argument(\"--gpt4-mini-only\", default=False, action=\"store_true\", help=\"Use only GPT-4o-mini\")\n parser.add_argument(\n \"--execution-strategy\",\n default=\"parallel\",\n type=str,\n help=\"The plan executor to use. One of sequential, pipelined, parallel\",\n )\n parser.add_argument(\n \"--sentinel-execution-strategy\",\n default=\"mab\",\n type=str,\n help=\"The sentinel execution strategy to use. One of mab or random\",\n )\n parser.add_argument(\n \"--policy\",\n default=\"maxquality\",\n type=str,\n help=\"One of 'mincost', 'mintime', 'maxquality'\",\n )\n parser.add_argument(\n \"--val-examples\",\n default=20,\n type=int,\n help=\"Number of validation examples to sample from\",\n )\n parser.add_argument(\n \"--model\",\n default=\"gpt-4o\",\n type=str,\n help=\"One of 'gpt-4o', 'gpt-4o-mini', 'llama'\",\n )\n parser.add_argument(\n \"--seed\",\n default=42,\n type=int,\n help=\"Seed used to initialize RNG for MAB sampling algorithm\",\n )\n parser.add_argument(\n \"--k\",\n default=6,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--j\",\n default=4,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--sample-budget\",\n default=100,\n type=int,\n help=\"Total sample budget in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--quality\",\n default=None,\n type=float,\n help=\"Quality threshold\",\n )\n parser.add_argument(\n \"--exp-name\",\n default=None,\n type=str,\n help=\"The experiment name.\",\n )\n\n args = parser.parse_args()\n\n # create directory for profiling data\n os.makedirs(\"mmqa-complex-data\", exist_ok=True)\n\n verbose = args.verbose\n progress = args.progress\n seed = args.seed\n val_examples = args.val_examples\n k = args.k\n j = args.j\n sample_budget = args.sample_budget\n execution_strategy = args.execution_strategy\n sentinel_execution_strategy = args.sentinel_execution_strategy\n exp_name = (\n f\"mmqa-complex-final-{sentinel_execution_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}\"\n if args.exp_name is None\n else args.exp_name\n )\n\n policy = pz.MaxQuality()\n if args.policy == \"mincost\":\n policy = pz.MinCost()\n elif args.policy == \"minlatency\":\n policy = pz.MinTime()\n elif args.quality is not None and args.policy == \"mincostatfixedquality\":\n policy = pz.MinCostAtFixedQuality(min_quality=args.quality)\n elif args.quality is not None and args.policy == \"minlatencyatfixedquality\":\n policy = pz.MinTimeAtFixedQuality(min_quality=args.quality)\n print(f\"USING POLICY: {policy}\")\n\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n # create the train and test dataset\n train_dataset = get_dataset(split=\"train\", shuffle=True, seed=seed, num_samples=val_examples)\n test_dataset = get_dataset(split=\"dev\", shuffle=True, seed=seed, num_samples=100)\n\n # create validator for MMQA\n validator = MMQAValidator(train_dataset)\n\n # create train datasets for questions, texts, tables, and images\n train_question_dataset = MMQAQuestionDataset(train_dataset)\n train_text_dataset = MMQATextDataset(train_dataset)\n train_table_dataset = MMQATableDataset(train_dataset)\n train_image_dataset = MMQAImageDataset(train_dataset)\n train_dataset = {\n train_question_dataset.id: train_question_dataset,\n train_text_dataset.id: train_text_dataset,\n train_table_dataset.id: train_table_dataset,\n train_image_dataset.id: train_image_dataset,\n }\n\n # construct plan\n test_question_dataset = MMQAQuestionDataset(test_dataset)\n print(f\"Test Question Dataset: {len(test_question_dataset)}\")\n test_text_dataset = MMQATextDataset(test_dataset)\n print(f\"Text Dataset: {len(test_text_dataset)}\")\n test_table_dataset = MMQATableDataset(test_dataset)\n print(f\"Table Dataset: {len(test_table_dataset)}\")\n test_image_dataset = MMQAImageDataset(test_dataset)\n print(f\"Image Dataset: {len(test_image_dataset)}\")\n\n text_plan = test_question_dataset.sem_map(mmqa_text_search_cols, depends_on=[\"question\"])\n text_plan = text_plan.sem_join(\n test_text_dataset,\n condition=\"The text snippet is relevant to the question based on the text search string.\",\n depends_on=[\"text_search_string\", \"text\"],\n how=\"left\",\n )\n text_plan = text_plan.groupby(pz.GroupBySig([\"qid\", \"question\", \"text_search_string\"], agg_funcs=[\"list\", \"list\"], agg_fields=[\"text_id\", \"text\"]))\n text_plan = text_plan.map(\n udf=lambda record: {\"text\": \"...\".join(record[\"list(text)\"]) if record[\"list(text)\"] != [None] else [None], **record},\n cols=[{\"name\": \"text\", \"type\": str, \"desc\": \"All relevant text snippets concatenated together.\"}],\n )\n text_plan = text_plan.project([\"qid\", \"question\", \"text\"])\n\n table_plan = test_question_dataset.sem_map(mmqa_table_search_cols, depends_on=[\"question\"])\n table_plan = table_plan.sem_join(\n test_table_dataset,\n condition=\"The table is relevant to the question based on the table search string.\",\n depends_on=[\"table_search_string\", \"table\"],\n how=\"left\",\n )\n table_plan = table_plan.groupby(pz.GroupBySig([\"qid\", \"question\", \"table_search_string\"], agg_funcs=[\"list\", \"list\"], agg_fields=[\"table_id\", \"table\"]))\n table_plan = table_plan.map(\n udf=lambda record: {\"table\": \"\\n\\n\".join(record[\"list(table)\"]) if record[\"list(table)\"] != [None] else [None], **record},\n cols=[{\"name\": \"table\", \"type\": str, \"desc\": \"All relevant tables concatenated together.\"}],\n )\n table_plan = table_plan.project([\"qid\", \"question\", \"table\"])\n\n image_plan = test_question_dataset.sem_map(mmqa_image_search_cols, depends_on=[\"question\"])\n image_plan = image_plan.sem_join(\n test_image_dataset,\n condition=\"The image is relevant to the question based on the image search string.\",\n depends_on=[\"image_search_string\", \"image\"],\n how=\"left\",\n )\n image_plan = image_plan.groupby(pz.GroupBySig([\"qid\", \"question\", \"image_search_string\"], agg_funcs=[\"list\", \"list\"], agg_fields=[\"image_id\", \"image\"]))\n image_plan = image_plan.map(\n udf=lambda record: {\"image\": record[\"list(image)\"] if record[\"list(image)\"] != [None] else [None], **record},\n cols=[{\"name\": \"image\", \"type\": list[ImageBase64], \"desc\": \"All relevant images.\"}],\n )\n image_plan = image_plan.project([\"qid\", \"question\", \"image\"])\n plan = text_plan.join(table_plan, on=[\"qid\", \"question\"]).join(image_plan, on=[\"qid\", \"question\"])\n plan = plan.sem_map(mmqa_answer_cols, depends_on=[\"question\", \"text\", \"table\", \"image\"])\n\n # execute pz plan\n config = pz.QueryProcessorConfig(\n policy=policy,\n optimizer_strategy=\"pareto\",\n sentinel_execution_strategy=sentinel_execution_strategy,\n execution_strategy=execution_strategy,\n use_final_op_quality=True,\n max_workers=64,\n verbose=verbose,\n available_models=[\n Model.GPT_4o_MINI,\n ],\n allow_bonded_query=True,\n allow_critic=True,\n allow_mixtures=True,\n allow_rag_reduction=True,\n progress=progress,\n k=k,\n j=j,\n sample_budget=sample_budget,\n seed=seed,\n exp_name=exp_name,\n )\n\n data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator)\n\n print(data_record_collection.to_df())\n data_record_collection.to_df().to_csv(f\"mmqa-complex-data/{exp_name}-output.csv\", index=False)\n\n # create filepaths for records and stats\n records_path = f\"mmqa-complex-data/{exp_name}-records.json\"\n stats_path = f\"mmqa-complex-data/{exp_name}-profiling.json\"\n\n # save record outputs\n record_jsons = []\n for record in data_record_collection:\n record_dict = record.to_dict()\n record_dict = {\n k: v\n for k, v in record_dict.items()\n if k in [\"qid\", \"question\", \"supporting_text_ids\", \"supporting_table_ids\", \"supporting_image_ids\", \"answers\"]\n }\n record_jsons.append(record_dict)\n\n with open(records_path, \"w\") as f:\n json.dump(record_jsons, f)\n\n # read the appropriate dataset\n dataset = []\n with open(\"data/MMQA_dev.jsonl\") as f:\n for line in f:\n dict_line = json.loads(line)\n if \"image\" in dict_line[\"metadata\"][\"modalities\"] and len(dict_line[\"metadata\"][\"modalities\"]) > 1:\n dataset.append(dict_line)\n\n # shuffle the questions for the given seed\n rng = np.random.default_rng(seed=seed)\n rng.shuffle(dataset)\n\n # trim to 100 samples\n dataset = dataset[:100]\n answer_dataset = []\n for item in dataset:\n answers = list(map(lambda elt: str(elt[\"answer\"]), item[\"answers\"]))\n answer_dataset.append({\n \"qid\": item[\"qid\"],\n \"gt_answers\": answers\n })\n\n # construction dataframe\n import pandas as pd\n answers_df = pd.DataFrame(answer_dataset)\n\n # get final plan str\n final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0]\n final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id]\n\n # write stats to disk\n stats_dict = {\n \"f1\": compute_f1(data_record_collection.to_df(), answers_df),\n \"optimization_time\": data_record_collection.execution_stats.optimization_time,\n \"optimization_cost\": data_record_collection.execution_stats.optimization_cost,\n \"plan_execution_time\": data_record_collection.execution_stats.plan_execution_time,\n \"plan_execution_cost\": data_record_collection.execution_stats.plan_execution_cost,\n \"total_execution_time\": data_record_collection.execution_stats.total_execution_time,\n \"total_execution_cost\": data_record_collection.execution_stats.total_execution_cost,\n \"plan_str\": final_plan_str,\n }\n print(f\"F1 IS: {stats_dict['f1']}\")\n\n with open(f\"mmqa-complex-data/{exp_name}-stats.json\", \"w\") as f:\n json.dump(stats_dict, f)\n" }, { "path": "abacus-research/mmqa-demo.py", "content": "import argparse\nimport base64\nimport json\nimport os\nimport string\nimport time\n\nimport chromadb\nimport numpy as np\nimport regex as re\nfrom chromadb.utils.embedding_functions import (\n SentenceTransformerEmbeddingFunction,\n)\nfrom chromadb.utils.embedding_functions.openai_embedding_function import (\n OpenAIEmbeddingFunction,\n)\n\nimport palimpzest as pz\nfrom palimpzest.constants import Model\nfrom palimpzest.core.lib.schemas import ImageBase64\n\nmmqa_entry_cols = [\n {\"name\": \"qid\", \"type\": str, \"desc\": \"The id of the MMQA question\"},\n {\"name\": \"question\", \"type\": str, \"desc\": \"The question which needs to be answered\"},\n]\n\nmmqa_text_cols = [\n {\"name\": \"supporting_text_ids\", \"type\": list[str], \"desc\": \"A list of text ids for text snippets which may support the question.\"},\n {\"name\": \"supporting_texts\", \"type\": list[str], \"desc\": \"A list of text snippets which may support the question.\"},\n]\n\nmmqa_table_cols = [\n {\"name\": \"supporting_table_ids\", \"type\": list[str], \"desc\": \"A list of table ids for tables which may support the question.\"},\n {\"name\": \"supporting_tables\", \"type\": list[str], \"desc\": \"A list of tables which may support the question.\"},\n]\n\nmmqa_image_cols = [\n {\"name\": \"supporting_image_ids\", \"type\": list[str], \"desc\": \"A list of image ids whose images may support the question.\"},\n {\"name\": \"supporting_images\", \"type\": list[ImageBase64], \"desc\": \"A list of images which may support the question.\"},\n]\n\nmmqa_answer_cols = [\n {\"name\": \"answers\", \"type\": list[str], \"desc\": \"The answer(s) to the question. Answer the question using the relevant information from gathered image(s), text(s), and table(s). Do not include any additional context or an explanation in your final list, simply list the entities asked for by the question\"},\n] # Return your answer as a JSON list of strings.\n\ndef get_json_from_answer(answer: str):\n \"\"\"\n This function parses an LLM response which is supposed to output a JSON object\n and optimistically searches for the substring containing the JSON object.\n \"\"\"\n # split off context / excess, which models sometimes output after answer\n answer = answer.split(\"Context:\")[0]\n answer = answer.split(\"# this is the answer\")[0]\n # trim the answer to only include the JSON array\n if not answer.strip().startswith(\"[\"):\n # Find the start index of the actual JSON string assuming the prefix is followed by the JSON array\n start_index = answer.find(\"[\")\n if start_index != -1:\n # Remove the prefix and any leading characters before the JSON starts\n answer = answer[start_index:]\n if not answer.strip().endswith(\"]\"):\n # Find the end index of the actual JSON string\n # assuming the suffix is preceded by the JSON object/array\n end_index = answer.rfind(\"]\")\n if end_index != -1:\n # Remove the suffix and any trailing characters after the JSON ends\n answer = answer[: end_index + 1]\n # Handle weird escaped values. I am not sure why the model\n # is returning these, but the JSON parser can't take them\n answer = answer.replace(r\"\\_\", \"_\")\n answer = answer.replace(\"\\\\n\", \"\\n\")\n # Remove https and http prefixes to not conflict with comment detection\n # Handle comments in the JSON response. Use regex from // until end of line\n answer = re.sub(r\"(? 1:\n dataset.append(dict_line)\n\n # shuffle the questions for the given seed\n if shuffle:\n rng = np.random.default_rng(seed=seed)\n rng.shuffle(dataset)\n\n # trim to number of samples\n self.dataset = dataset[:num_samples]\n self.num_samples = num_samples\n self.shuffle = shuffle\n self.seed = seed\n\n # compute qid to label mapping\n self.qid_to_labels = self._compute_qid_to_labels()\n\n def _compute_qid_to_labels(self) -> dict:\n \"\"\"Compute the label for a MMQA question given its entry in the dataset.\"\"\"\n qid_to_labels = {}\n for entry in self.dataset:\n # get the answers\n answers = [answer[\"answer\"] for answer in entry[\"answers\"]]\n supporting_text_doc_ids = [context[\"doc_id\"] for context in entry[\"supporting_context\"] if context[\"doc_part\"] == \"text\"]\n supporting_table_doc_ids = [context[\"doc_id\"] for context in entry[\"supporting_context\"] if context[\"doc_part\"] == \"table\"]\n supporting_image_doc_ids = [context[\"doc_id\"] for context in entry[\"supporting_context\"] if context[\"doc_part\"] == \"image\"]\n\n # NOTE: inside the optimizer, our qualities will effectively be divided by two,\n # because we are not providing a label for supporting texts, tables, and images,\n # however this should be okay b/c it will affect all records equally\n label_dict = {\n \"answers\": answers,\n \"supporting_text_ids\": supporting_text_doc_ids,\n \"supporting_table_ids\": supporting_table_doc_ids,\n \"supporting_image_ids\": supporting_image_doc_ids,\n \"supporting_texts\": [],\n \"supporting_tables\": [],\n \"supporting_images\": [],\n }\n qid_to_labels[entry[\"qid\"]] = label_dict\n\n return qid_to_labels\n\n def recall(self, preds: list | None, targets: list):\n if preds is None or len(targets) == 0:\n return 0.0\n\n tp, fn = 0, 0\n try:\n if isinstance(preds, list) and len(preds) > 0 and isinstance(preds[0], list):\n preds = preds[0]\n\n # compute recall of retrieved ids and return\n preds = [str(pred).lower() for pred in preds]\n targets = [str(target).lower() for target in targets]\n remove_tokens = [c for c in string.punctuation if c != \"/\"]\n for token in remove_tokens:\n preds = [pred.replace(token, \"\") for pred in preds]\n targets = [target.replace(token, \"\") for target in targets]\n\n\n for target in targets:\n if target in preds:\n tp += 1\n else:\n fn += 1\n\n return tp / (tp + fn)\n\n except Exception:\n os.makedirs(\"mmqa-recall-eval-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"mmqa-recall-eval-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def f1(self, preds: list | None, targets: list):\n if preds is None or len(targets) == 0:\n return 0.0\n\n tp, fp, fn = 0, 0, 0\n try:\n if isinstance(preds, list) and len(preds) > 0 and isinstance(preds[0], list):\n preds = preds[0]\n\n # compute recall of retrieved ids and return\n preds = [str(pred).lower() for pred in preds]\n targets = [str(target).lower() for target in targets]\n\n remove_tokens = [c for c in string.punctuation if c != \"/\"]\n for token in remove_tokens:\n preds = [pred.replace(token, \"\") for pred in preds]\n targets = [target.replace(token, \"\") for target in targets]\n\n for pred in preds:\n if pred in targets:\n tp += 1\n else:\n fp += 1\n for target in targets:\n if target not in preds:\n fn += 1\n\n # compute overall f1 score and return\n recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n\n return f1\n\n except Exception:\n os.makedirs(\"mmqa-recall-eval-errors\", exist_ok=True)\n ts = time.time()\n with open(f\"mmqa-recall-eval-errors/error-{ts}.txt\", \"w\") as f:\n f.write(str(preds))\n return 0.0\n\n def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n preds = output.get(\"answers\")\n targets = self.qid_to_labels[str(input_record[\"qid\"])][\"answers\"]\n return self.f1(preds, targets)\n\n def topk_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n if \"supporting_text_ids\" in fields:\n preds = output.get(\"supporting_text_ids\")\n targets = self.qid_to_labels[str(input_record[\"qid\"])][\"supporting_text_ids\"]\n return self.recall(preds, targets)\n elif \"supporting_table_ids\" in fields:\n preds = output.get(\"supporting_table_ids\")\n targets = self.qid_to_labels[str(input_record[\"qid\"])][\"supporting_table_ids\"]\n return self.recall(preds, targets)\n elif \"supporting_image_ids\" in fields:\n preds = output.get(\"supporting_image_ids\")\n targets = self.qid_to_labels[str(input_record[\"qid\"])][\"supporting_image_ids\"]\n return self.recall(preds, targets)\n else:\n raise NotImplementedError(f\"Validator.topk_score_fn not implemented for fields {fields}.\")\n\n\nclass MMQADataset(pz.IterDataset):\n def __init__(\n self,\n num_samples: int = 5,\n split: str = \"dev\",\n shuffle: bool = False,\n seed: int = 42,\n ):\n super().__init__(id=\"mmqa\", schema=mmqa_entry_cols)\n\n # read the appropriate dataset\n dataset = []\n with open(f\"data/MMQA_{split}.jsonl\") as f:\n for line in f:\n dict_line = json.loads(line)\n if \"image\" in dict_line[\"metadata\"][\"modalities\"] and len(dict_line[\"metadata\"][\"modalities\"]) > 1:\n dataset.append(dict_line)\n\n # shuffle the questions for the given seed\n if shuffle:\n rng = np.random.default_rng(seed=seed)\n rng.shuffle(dataset)\n\n # trim to number of samples\n self.dataset = dataset[:num_samples]\n self.num_samples = num_samples\n self.shuffle = shuffle\n self.seed = seed\n self.split = split\n\n def __len__(self):\n return len(self.dataset)\n\n def __getitem__(self, idx: int):\n # get entry\n entry = self.dataset[idx]\n\n # get input fields\n qid = entry[\"qid\"]\n question = entry[\"question\"]\n\n # create item with fields\n item = {\"qid\": qid, \"question\": question}\n\n return item\n\n\ndef compute_f1(final_df, answers_df):\n merged_df = final_df.merge(answers_df, on=\"qid\", how=\"left\")\n tp, fp, fn = 0, 0, 0\n for _, row in merged_df.iterrows():\n targets = [str(target).lower() for target in row[\"gt_answers\"]]\n preds = row[\"answers\"]\n if isinstance(preds, str):\n try:\n # convert single quotes to double quotes before parsing for JSON\n preds = preds.replace(\"'\", '\"')\n # try parsing preds as JSON list and cast everything to str to match targets\n preds = get_json_from_answer(preds)\n if isinstance(preds, list) and len(preds) > 0 and isinstance(preds[0], list):\n preds = preds[0]\n preds = [str(pred).lower() for pred in preds]\n except Exception:\n # if that fails, give it a shot as a singleton answer that the LLM failed to wrap in a list\n preds = [str(preds).lower()]\n remove_tokens = [c for c in string.punctuation if c != \"/\"]\n for token in remove_tokens:\n preds = [pred.replace(token, \"\") for pred in preds]\n targets = [target.replace(token, \"\") for target in targets]\n elif isinstance(preds, list):\n if isinstance(preds, list) and len(preds) > 0 and isinstance(preds[0], list):\n preds = preds[0]\n preds = [str(pred).lower() for pred in preds]\n remove_tokens = [c for c in string.punctuation if c != \"/\"]\n for token in remove_tokens:\n preds = [pred.replace(token, \"\") for pred in preds]\n targets = [target.replace(token, \"\") for target in targets]\n else:\n preds = []\n for pred in preds:\n if pred in targets:\n tp += 1\n else:\n fp += 1\n for target in targets:\n if target not in preds:\n fn += 1\n # compute overall f1 score and return\n recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n return f1\n\n\nif __name__ == \"__main__\":\n # parse arguments\n parser = argparse.ArgumentParser(description=\"Run a simple demo\")\n parser.add_argument(\"--verbose\", default=False, action=\"store_true\", help=\"Print verbose output\")\n parser.add_argument(\"--progress\", default=False, action=\"store_true\", help=\"Print progress output\")\n parser.add_argument(\"--gpt4-mini-only\", default=False, action=\"store_true\", help=\"Use only GPT-4o-mini\")\n parser.add_argument(\n \"--execution-strategy\",\n default=\"parallel\",\n type=str,\n help=\"The plan executor to use. One of sequential, pipelined, parallel\",\n )\n parser.add_argument(\n \"--sentinel-execution-strategy\",\n default=\"mab\",\n type=str,\n help=\"The sentinel execution strategy to use. One of mab or random\",\n )\n parser.add_argument(\n \"--policy\",\n default=\"maxquality\",\n type=str,\n help=\"One of 'mincost', 'mintime', 'maxquality'\",\n )\n parser.add_argument(\n \"--val-examples\",\n default=20,\n type=int,\n help=\"Number of validation examples to sample from\",\n )\n parser.add_argument(\n \"--model\",\n default=\"gpt-4o\",\n type=str,\n help=\"One of 'gpt-4o', 'gpt-4o-mini', 'llama'\",\n )\n parser.add_argument(\n \"--seed\",\n default=42,\n type=int,\n help=\"Seed used to initialize RNG for MAB sampling algorithm\",\n )\n parser.add_argument(\n \"--k\",\n default=6,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--j\",\n default=4,\n type=int,\n help=\"Number of columns to sample in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--sample-budget\",\n default=100,\n type=int,\n help=\"Total sample budget in Random Sampling or MAB sentinel execution\",\n )\n parser.add_argument(\n \"--quality\",\n default=None,\n type=float,\n help=\"Quality threshold\",\n )\n parser.add_argument(\n \"--exp-name\",\n default=None,\n type=str,\n help=\"The experiment name.\",\n )\n\n args = parser.parse_args()\n\n # create directory for profiling data\n os.makedirs(\"opt-profiling-data\", exist_ok=True)\n\n verbose = args.verbose\n progress = args.progress\n seed = args.seed\n val_examples = args.val_examples\n k = args.k\n j = args.j\n sample_budget = args.sample_budget\n execution_strategy = args.execution_strategy\n sentinel_execution_strategy = args.sentinel_execution_strategy\n exp_name = (\n f\"mmqa-final-{sentinel_execution_strategy}-k{k}-j{j}-budget{sample_budget}-seed{seed}\"\n if args.exp_name is None\n else args.exp_name\n )\n\n policy = pz.MaxQuality()\n if args.policy == \"mincost\":\n policy = pz.MinCost()\n elif args.policy == \"minlatency\":\n policy = pz.MinTime()\n elif args.quality is not None and args.policy == \"mincostatfixedquality\":\n policy = pz.MinCostAtFixedQuality(min_quality=args.quality)\n elif args.quality is not None and args.policy == \"minlatencyatfixedquality\":\n policy = pz.MinTimeAtFixedQuality(min_quality=args.quality)\n print(f\"USING POLICY: {policy}\")\n\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n # create validator for MMQA\n validator = MMQAValidator(num_samples=val_examples, shuffle=True, seed=seed)\n\n # create datasets for MMQA\n train_dataset = MMQADataset(split=\"train\", num_samples=val_examples, shuffle=True, seed=seed)\n train_dataset = {train_dataset.id: train_dataset}\n\n # load index [text-embedding-3-small]\n chroma_client = chromadb.PersistentClient(\".chroma-mmqa\")\n openai_ef = OpenAIEmbeddingFunction(\n api_key=os.environ[\"OPENAI_API_KEY\"],\n model_name=\"text-embedding-3-small\",\n )\n sentence_transformer_ef = SentenceTransformerEmbeddingFunction(\n model_name=\"clip-ViT-B-32\"\n )\n text_index = chroma_client.get_collection(\"mmqa-texts\", embedding_function=openai_ef)\n table_index = chroma_client.get_collection(\"mmqa-tables\", embedding_function=openai_ef)\n image_index = chroma_client.get_collection(\"mmqa-images\", embedding_function=sentence_transformer_ef)\n\n def get_results_and_ids(index: chromadb.Collection, query: list[list[float]], n_results: int, image=False) -> tuple[list[str]]:\n # execute query with embeddings\n results = index.query(query, n_results=n_results)\n\n # get list of result terms with their cosine similarity scores\n final_results = []\n for query_doc_ids, query_docs, query_distances in zip(results[\"ids\"], results[\"documents\"], results[\"distances\"]):\n for doc_id, doc, dist in zip(query_doc_ids, query_docs, query_distances):\n cosine_similarity = 1 - dist\n final_results.append({\"content\": doc, \"id\": doc_id, \"similarity\": cosine_similarity})\n\n # sort the results by similarity score\n sorted_results = sorted(final_results, key=lambda result: result[\"similarity\"], reverse=True)\n\n # remove duplicates\n sorted_results_set = set()\n final_sorted_results, final_sorted_result_ids = [], []\n for result in sorted_results:\n if result[\"content\"] not in sorted_results_set:\n sorted_results_set.add(result[\"content\"])\n final_sorted_results.append(result[\"content\"])\n final_sorted_result_ids.append(result[\"id\"])\n\n # return the top-k similar results and generation stats\n return final_sorted_results[:n_results], final_sorted_result_ids[:n_results]\n\n def text_search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]:\n # execute query with embeddings\n results, result_ids = get_results_and_ids(index, query, n_results=k)\n return {\"supporting_texts\": results, \"supporting_text_ids\": result_ids}\n\n def table_search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]:\n # execute query with embeddings\n results, result_ids = get_results_and_ids(index, query, n_results=k)\n return {\"supporting_tables\": results, \"supporting_table_ids\": result_ids}\n\n def image_search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]:\n # limit max number of results to 5\n # k = min(k, 5)\n\n # execute query with embeddings\n _, result_ids = get_results_and_ids(index, query, n_results=k, image=True)\n possible_endings = {'.JPG', '.png', '.jpeg', '.jpg', '.tif', '.JPEG', '.tiff', '.PNG', '.Jpg', '.gif'}\n\n results = []\n for image_id in result_ids:\n # find the correct image file\n for ending in possible_endings:\n if os.path.exists(f\"/ssd1/mdrusso/mmqa-images/{image_id}{ending}\"):\n image_id += ending\n break\n\n # load image from disk\n with open(f\"/ssd1/mdrusso/mmqa-images/{image_id}\", \"rb\") as f:\n base64_image_str = base64.b64encode(f.read()).decode(\"utf-8\")\n results.append(base64_image_str)\n\n return {\"supporting_images\": results, \"supporting_image_ids\": result_ids}\n\n # construct plan\n plan = MMQADataset(split=\"dev\", num_samples=100, shuffle=True, seed=seed)\n plan = plan.sem_topk(\n index=text_index,\n search_func=text_search_func,\n search_attr=\"question\",\n output_attrs=mmqa_text_cols,\n )\n plan = plan.sem_topk(\n index=table_index,\n search_func=table_search_func,\n search_attr=\"question\",\n output_attrs=mmqa_table_cols,\n )\n plan = plan.sem_topk(\n index=image_index,\n search_func=image_search_func,\n search_attr=\"question\",\n output_attrs=mmqa_image_cols,\n )\n plan = plan.sem_map(mmqa_answer_cols, depends_on=[\"question\", \"supporting_texts\", \"supporting_tables\", \"supporting_images\"])\n\n # execute pz plan\n config = pz.QueryProcessorConfig(\n policy=policy,\n optimizer_strategy=\"pareto\",\n sentinel_execution_strategy=sentinel_execution_strategy,\n execution_strategy=execution_strategy,\n use_final_op_quality=True,\n max_workers=1,\n verbose=verbose,\n available_models=[\n Model.GPT_4o_MINI,\n ],\n allow_bonded_query=True,\n allow_critic=True,\n allow_mixtures=True,\n allow_rag_reduction=True,\n progress=progress,\n k=k,\n j=j,\n sample_budget=sample_budget,\n seed=seed,\n exp_name=exp_name,\n )\n\n data_record_collection = plan.optimize_and_run(config=config, train_dataset=train_dataset, validator=validator)\n\n print(data_record_collection.to_df())\n data_record_collection.to_df().to_csv(f\"opt-profiling-data/{exp_name}-output.csv\", index=False)\n\n # create filepaths for records and stats\n records_path = f\"opt-profiling-data/{exp_name}-records.json\"\n stats_path = f\"opt-profiling-data/{exp_name}-profiling.json\"\n\n # save record outputs\n record_jsons = []\n for record in data_record_collection:\n record_dict = record.to_dict()\n record_dict = {\n k: v\n for k, v in record_dict.items()\n if k in [\"qid\", \"question\", \"supporting_text_ids\", \"supporting_table_ids\", \"supporting_image_ids\", \"answers\"]\n }\n record_jsons.append(record_dict)\n\n with open(records_path, \"w\") as f:\n json.dump(record_jsons, f)\n\n # read the appropriate dataset\n dataset = []\n with open(\"data/MMQA_dev.jsonl\") as f:\n for line in f:\n dict_line = json.loads(line)\n if \"image\" in dict_line[\"metadata\"][\"modalities\"] and len(dict_line[\"metadata\"][\"modalities\"]) > 1:\n dataset.append(dict_line)\n\n # shuffle the questions for the given seed\n rng = np.random.default_rng(seed=seed)\n rng.shuffle(dataset)\n\n # trim to 100 samples\n dataset = dataset[:100]\n answer_dataset = []\n for item in dataset:\n answers = list(map(lambda elt: str(elt[\"answer\"]), item[\"answers\"]))\n answer_dataset.append({\n \"qid\": item[\"qid\"],\n \"gt_answers\": answers\n })\n\n # construction dataframe\n import pandas as pd\n answers_df = pd.DataFrame(answer_dataset)\n\n # get final plan str\n final_plan_id = list(data_record_collection.execution_stats.plan_stats.keys())[0]\n final_plan_str = data_record_collection.execution_stats.plan_strs[final_plan_id]\n\n # write stats to disk\n stats_dict = {\n \"f1\": compute_f1(data_record_collection.to_df(), answers_df),\n \"optimization_time\": data_record_collection.execution_stats.optimization_time,\n \"optimization_cost\": data_record_collection.execution_stats.optimization_cost,\n \"plan_execution_time\": data_record_collection.execution_stats.plan_execution_time,\n \"plan_execution_cost\": data_record_collection.execution_stats.plan_execution_cost,\n \"total_execution_time\": data_record_collection.execution_stats.total_execution_time,\n \"total_execution_cost\": data_record_collection.execution_stats.total_execution_cost,\n \"plan_str\": final_plan_str,\n }\n print(f\"F1 IS: {stats_dict['f1']}\")\n\n with open(f\"opt-profiling-data/{exp_name}-stats.json\", \"w\") as f:\n json.dump(stats_dict, f)\n" }, { "path": "abacus-research/run_ablation_study.sh", "content": "#!/bin/bash\n\nfor seed in {0..9}\ndo\n for priors in none naive sample\n do\n for sentinel in mab random\n do\n for opt in pareto greedy\n do\n k=6\n j=4\n if [[ $sentinel -eq \"random\" ]]; then\n j=8\n fi\n priors_file=\"none\"\n if [[ $priors -eq \"naive\" ]]; then\n priors_file=\"cheap-priors.json\"\n elif [[ $priors -eq \"sample\" ]]; then\n priors_file=\"biodex-priors.json\"\n fi\n\n exp_name=\"ablation-${priors}-${sentinel}-${opt}-seed${seed}\"\n FILE=\"ablation-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- priors: ${priors} (${priors_file}) -- sentinel: ${sentinel} -- k: ${k} -- j: ${j} -- opt: ${opt}\"\n python biodex-ablation.py --priors-file $priors_file --k $k --j $j --sample-budget 150 --optimizer-strategy $opt --sentinel-execution-strategy $sentinel --seed $seed --exp-name $exp_name\n fi\n done\n done\n done\ndone\n" }, { "path": "abacus-research/run_biodex.sh", "content": "#!/bin/bash\n\nfor seed in {0..9}\ndo\n echo \"Running Seed: ${seed}\"\n exp_name=\"biodex-final-mab-k6-j4-budget150-seed${seed}\"\n python biodex-demo.py --progress --policy maxquality --val-examples 20 --k 6 --j 4 --sample-budget 150 --seed $seed --exp-name $exp_name --gpt4-mini-only\ndone\n" }, { "path": "abacus-research/run_biodex_cascades.sh", "content": "#!/bin/bash\n\n\nfor seed in {0..9}\ndo\n for budget in 150 300 450\n do\n for strategy in \"greedy\" \"pareto\"\n do\n cost=0.5\n k=0\n j=0\n if [[ $budget -eq 150 ]]; then\n k=6\n j=4\n elif [[ $budget -eq 300 ]]; then\n k=24\n j=5\n elif [[ $budget -eq 450 ]]; then\n k=48\n j=6\n fi\n # no priors\n exp_name=\"biodex-${strategy}-cost${cost}-budget${budget}-k${k}-j${j}-seed${seed}\"\n FILE=\"pareto-cascades-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- cost: ${cost} -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}\"\n python biodex-pareto-cascades.py --progress --k $k --j $j --sample-budget $budget --optimizer-strategy $strategy --cost $cost --seed $seed --exp-name $exp_name\n fi\n\n # sample priors\n exp_name=\"biodex-${strategy}-cost${cost}-with-priors-budget${budget}-k${k}-j${j}-seed${seed}\"\n FILE=\"pareto-cascades-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- cost: ${cost} -- SAMPLE PRIORS -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}\"\n python biodex-pareto-cascades.py --progress --priors-file biodex-priors-cascades.json --k $k --j $j --sample-budget $budget --optimizer-strategy $strategy --cost $cost --seed $seed --exp-name $exp_name\n fi\n\n # naive priors\n exp_name=\"biodex-${strategy}-cost${cost}-cheap-priors-budget${budget}-k${k}-j${j}-seed${seed}\"\n FILE=\"pareto-cascades-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- cost: ${cost} -- CHEAP PRIORS -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}\"\n python biodex-pareto-cascades.py --progress --priors-file cheap-priors-cascades.json --k $k --j $j --sample-budget $budget --optimizer-strategy $strategy --cost $cost --seed $seed --exp-name $exp_name\n fi\n done\n done\ndone\n" }, { "path": "abacus-research/run_biodex_cost_threshold.sh", "content": "#!/bin/bash\n\n\nfor cost in 1.0 2.0 4.0 8.0 999.99\ndo\n for seed in {0..9}\n do\n # set variables\n budget=450\n k=48\n j=3\n\n # no priors\n exp_name=\"biodex-pareto-cost${cost}-budget${budget}-k${k}-j${j}-seed${seed}\"\n FILE=\"max-quality-at-cost-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- cost: ${cost} -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}\"\n python biodex-max-quality-at-cost.py --progress --k $k --j $j --sample-budget $budget --cost $cost --seed $seed --exp-name $exp_name\n fi\n\n # sample priors\n exp_name=\"biodex-pareto-cost${cost}-with-priors-budget${budget}-k${k}-j${j}-seed${seed}\"\n FILE=\"max-quality-at-cost-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- cost: ${cost} -- SAMPLE PRIORS -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}\"\n python biodex-max-quality-at-cost.py --progress --priors-file biodex-priors.json --k $k --j $j --sample-budget $budget --cost $cost --seed $seed --exp-name $exp_name\n fi\n done\ndone\n" }, { "path": "abacus-research/run_biodex_min_cost_latency.sh", "content": "#!/bin/bash\n\n\nfor policy in \"mincost\" \"minlatency\"\ndo\n for seed in {0..9}\n do\n # set variables\n budget=150\n k=6\n j=4\n\n echo \"Running Seed: ${seed}\"\n exp_name=\"biodex-final-${policy}-k6-j4-budget150-seed${seed}\"\n python biodex-demo.py --progress --policy $policy --val-examples 20 --k 6 --j 4 --sample-budget 150 --seed $seed --exp-name $exp_name --gpt4-mini-only\n\n done\ndone\n" }, { "path": "abacus-research/run_biodex_priors.sh", "content": "#!/bin/bash\n\nfor sample_budget in 10 20 50 100\ndo\n for seed in {0..9}\n do\n k=0\n j=0\n if [[ $sample_budget -eq 10 ]]; then\n k=2\n j=2\n elif [[ $sample_budget -eq 20 ]]; then\n k=2\n j=2\n elif [[ $sample_budget -eq 50 ]]; then\n k=3\n j=3\n elif [[ $sample_budget -eq 100 ]]; then\n k=4\n j=4\n fi\n\n # run without priors\n exp_name=\"biodex-no-priors-k${k}-j${j}-budget${sample_budget}-seed${seed}\"\n FILE=\"opt-profiling-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- priors: NO PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}\"\n python biodex-demo.py --progress --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name\n fi\n\n # run with sample-based priors\n exp_name=\"biodex-with-priors-k${k}-j${j}-budget${sample_budget}-seed${seed}\"\n FILE=\"opt-profiling-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- priors: WITH PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}\"\n python biodex-demo.py --progress --priors-file biodex-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name\n fi\n\n # run with cheap priors \n exp_name=\"biodex-cheap-priors-k${k}-j${j}-budget${sample_budget}-seed${seed}\"\n FILE=\"opt-profiling-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- priors: CHEAP PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}\"\n python biodex-demo.py --progress --priors-file cheap-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name\n fi\n done\ndone\n" }, { "path": "abacus-research/run_biodex_priors_constrained.sh", "content": "#!/bin/bash\n\nfor sample_budget in 10 20 50 100\ndo\n for seed in {0..9}\n do\n k=0\n j=0\n if [[ $sample_budget -eq 10 ]]; then\n k=4\n j=1\n elif [[ $sample_budget -eq 20 ]]; then\n k=4\n j=1\n elif [[ $sample_budget -eq 50 ]]; then\n k=4\n j=2\n elif [[ $sample_budget -eq 100 ]]; then\n k=5\n j=3\n fi\n\n # run without priors\n exp_name=\"biodex-no-priors-constrained-k${k}-j${j}-budget${sample_budget}-seed${seed}\"\n FILE=\"opt-profiling-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- priors: NO PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}\"\n python biodex-demo.py --constrained --progress --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name\n fi\n\n # run with sample-based priors\n exp_name=\"biodex-with-priors-constrained-k${k}-j${j}-budget${sample_budget}-seed${seed}\"\n FILE=\"opt-profiling-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- priors: WITH PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}\"\n python biodex-demo.py --constrained --progress --priors-file biodex-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name\n fi\n\n # run with cheap priors \n exp_name=\"biodex-cheap-priors-constrained-k${k}-j${j}-budget${sample_budget}-seed${seed}\"\n FILE=\"opt-profiling-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- priors: CHEAP PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}\"\n python biodex-demo.py --constrained --progress --priors-file cheap-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name\n fi\n done\ndone\n" }, { "path": "abacus-research/run_cuad.sh", "content": "#!/bin/bash\n\nfor seed in {0..9}\ndo\n policy=\"maxquality\"\n echo \"Running Seed: ${seed} -- policy: ${policy}\"\n exp_name=\"cuad-${policy}-k6-j4-budget50-seed${seed}\"\n python cuad-demo.py --k 6 --j 4 --sample-budget 50 --seed $seed --exp-name $exp_name --gpt4-mini-only\ndone\n" }, { "path": "abacus-research/run_cuad_cost_threshold.sh", "content": "#!/bin/bash\n\n\nfor cost in 1.0 2.0 4.0 8.0 999.99\ndo\n for seed in {0..9}\n do\n # set variables\n budget=300\n k=60\n j=5\n\n # no priors\n exp_name=\"cuad-pareto-cost${cost}-budget${budget}-k${k}-j${j}-seed${seed}\"\n FILE=\"max-quality-at-cost-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- cost: ${cost} -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}\"\n python cuad-max-quality-at-cost.py --k $k --j $j --sample-budget $budget --cost $cost --seed $seed --exp-name $exp_name\n fi\n\n # sample priors\n exp_name=\"cuad-pareto-cost${cost}-with-priors-budget${budget}-k${k}-j${j}-seed${seed}\"\n FILE=\"max-quality-at-cost-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- cost: ${cost} -- SAMPLE PRIORS -- budget: ${budget} -- k: ${k} -- j: ${j} -- strategy: ${strategy}\"\n python cuad-max-quality-at-cost.py --priors-file cuad-priors.json --k $k --j $j --sample-budget $budget --cost $cost --seed $seed --exp-name $exp_name\n fi\n\n done\ndone\n" }, { "path": "abacus-research/run_cuad_min_cost_latency.sh", "content": "#!/bin/bash\n\nfor policy in \"mincost\" \"minlatency\"\ndo\n for seed in {0..9}\n do\n echo \"Running Seed: ${seed}\"\n exp_name=\"cuad-final-${policy}-k6-j4-budget50-seed${seed}\"\n python cuad-demo.py --policy $policy --k 6 --j 4 --sample-budget 50 --seed $seed --exp-name $exp_name --gpt4-mini-only\n done\ndone\n" }, { "path": "abacus-research/run_cuad_priors.sh", "content": "#!/bin/bash\n\nfor sample_budget in 5 10 20 50\ndo\n for seed in {0..9}\n do\n k=0\n j=0\n if [[ $sample_budget -eq 5 ]]; then\n k=2\n j=3\n elif [[ $sample_budget -eq 10 ]]; then\n k=3\n j=2\n elif [[ $sample_budget -eq 20 ]]; then\n k=3\n j=3\n elif [[ $sample_budget -eq 50 ]]; then\n k=6\n j=4\n fi\n\n # run without priors\n exp_name=\"cuad-no-priors-k${k}-j${j}-budget${sample_budget}-seed${seed}\"\n FILE=\"opt-profiling-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- priors: NO PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}\"\n python cuad-demo.py --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name\n fi\n\n # run with sample based priors\n exp_name=\"cuad-with-priors-k${k}-j${j}-budget${sample_budget}-seed${seed}\"\n FILE=\"opt-profiling-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- priors: WITH PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}\"\n python cuad-demo.py --priors-file cuad-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name\n fi\n\n # run with cheap priors \n exp_name=\"cuad-cheap-priors-k${k}-j${j}-budget${sample_budget}-seed${seed}\"\n FILE=\"opt-profiling-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- priors: CHEAP PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}\"\n python cuad-demo.py --priors-file cheap-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name\n fi\n done\ndone\n" }, { "path": "abacus-research/run_cuad_priors_constrained.sh", "content": "#!/bin/bash\n\nfor sample_budget in 5 10 20 50\ndo\n for seed in {0..9}\n do\n k=0\n j=0\n if [[ $sample_budget -eq 5 ]]; then\n k=2\n j=3\n elif [[ $sample_budget -eq 10 ]]; then\n k=3\n j=2\n elif [[ $sample_budget -eq 20 ]]; then\n k=3\n j=3\n elif [[ $sample_budget -eq 50 ]]; then\n k=6\n j=4\n fi\n\n # run without priors\n exp_name=\"cuad-no-priors-constrained-k${k}-j${j}-budget${sample_budget}-seed${seed}\"\n FILE=\"opt-profiling-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- priors: NO PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}\"\n python cuad-demo.py --constrained --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name\n fi\n\n # run with sample based priors\n exp_name=\"cuad-with-priors-constrained-k${k}-j${j}-budget${sample_budget}-seed${seed}\"\n FILE=\"opt-profiling-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- priors: WITH PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}\"\n python cuad-demo.py --constrained --priors-file cuad-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name\n fi\n\n # run with cheap priors \n exp_name=\"cuad-cheap-priors-constrained-k${k}-j${j}-budget${sample_budget}-seed${seed}\"\n FILE=\"opt-profiling-data/${exp_name}-metrics.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- priors: CHEAP PRIORS -- k: ${k} -- j: ${j} -- budget: ${sample_budget}\"\n python cuad-demo.py --constrained --priors-file cheap-priors.json --k $k --j $j --sample-budget $sample_budget --seed $seed --exp-name $exp_name\n fi\n done\ndone\n" }, { "path": "abacus-research/run_mmqa.sh", "content": "#!/bin/bash\n\nfor seed in {0..9}\ndo\n echo \"Running Seed: ${seed}\"\n exp_name=\"mmqa-final-mab-k6-j4-budget150-seed${seed}\"\n python mmqa-demo.py --progress --k 6 --j 4 --sample-budget 150 --seed $seed --exp-name $exp_name --gpt4-mini-only\ndone\n" }, { "path": "abacus-research/run_mmqa_complex.sh", "content": "#!/bin/bash\n\n# for seed in {0..9}\n# Lotus error'ed on seed 5 and 7, so we limit to these seeds only for a consistent comparison\nfor seed in 0 1 2 3 4 6 8 9\ndo\n policy=\"maxquality\"\n exp_name=\"mmqa-complex-${policy}-k6-j4-budget350-seed${seed}\"\n FILE=\"mmqa-complex-data/${exp_name}-stats.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- ${policy}\"\n python mmqa-complex-demo.py --progress --k 6 --j 4 --sample-budget 350 --seed $seed --exp-name $exp_name --gpt4-mini-only\n fi\ndone\n" }, { "path": "abacus-research/run_mmqa_complex_min_cost_latency.sh", "content": "#!/bin/bash\n\n# for seed in {0..9}\n# Lotus error'ed on seed 5 and 7, so we limit to these seeds only for a consistent comparison\nfor seed in 0 1 2 3 4 6 8 9\ndo\n policy=\"mincost\"\n exp_name=\"mmqa-complex-${policy}-k6-j4-budget350-seed${seed}\"\n FILE=\"mmqa-complex-data/${exp_name}-stats.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- ${policy}\"\n python mmqa-complex-demo.py --progress --k 6 --j 4 --sample-budget 350 --policy $policy --seed $seed --exp-name $exp_name --gpt4-mini-only\n fi\n\n policy=\"minlatency\"\n exp_name=\"mmqa-complex-${policy}-k6-j4-budget350-seed${seed}\"\n FILE=\"mmqa-complex-data/${exp_name}-stats.json\"\n if [ -f $FILE ]; then\n echo \"Skipping because $FILE exists.\"\n else\n echo \"Running Seed: ${seed} -- ${policy}\"\n python mmqa-complex-demo.py --progress --k 6 --j 4 --sample-budget 350 --policy $policy --seed $seed --exp-name $exp_name --gpt4-mini-only\n fi\ndone\n" }, { "path": "abacus-research/run_mmqa_min_cost_latency.sh", "content": "#!/bin/bash\n\nfor policy in \"mincost\" \"minlatency\"\ndo\n for seed in {0..9}\n do\n echo \"Running Seed: ${seed}\"\n exp_name=\"mmqa-final-${policy}-k6-j4-budget150-seed${seed}\"\n python mmqa-demo.py --progress --policy $policy --k 6 --j 4 --sample-budget 150 --seed $seed --exp-name $exp_name --gpt4-mini-only\n done\ndone\n" }, { "path": "abacus-research/score_biodex.py", "content": "import json\n\nimport numpy as np\n\n\ndef compute_final_metrics(metric: str, dir: str, exp_base_name: str):\n qualities = []\n opt_costs, run_costs = [], []\n opt_times, run_times = [], []\n total_costs, total_times = [], []\n print(f\"--- {metric} ---\")\n for seed in range(10):\n exp_name = f\"{exp_base_name}-seed{seed}\"\n with open(f\"{dir}/{exp_name}-metrics.json\") as f:\n metrics = json.load(f)\n qualities.append(metrics[\"rp@5\"])\n opt_costs.append(metrics[\"optimization_cost\"])\n opt_times.append(metrics[\"optimization_time\"])\n run_costs.append(metrics[\"plan_execution_cost\"])\n run_times.append(metrics[\"plan_execution_time\"])\n total_costs.append(metrics[\"total_execution_cost\"])\n total_times.append(metrics[\"total_execution_time\"])\n \n print(f\"Opt. Cost: {np.mean(opt_costs):.3f} +/- {np.std(opt_costs):.3f}\")\n print(f\"Opt. Time: {np.mean(opt_times):.3f} +/- {np.std(opt_times):.3f}\")\n print(f\"Run Cost: {np.mean(run_costs):.3f} +/- {np.std(run_costs):.3f}\")\n print(f\"Run Time: {np.mean(run_times):.3f} +/- {np.std(run_times):.3f}\")\n print(f\"Total Cost: {np.mean(total_costs):.3f} +/- {np.std(total_costs):.3f}\")\n print(f\"Total Time: {np.mean(total_times):.3f} +/- {np.std(total_times):.3f}\")\n print(f\"Quality: {np.mean(qualities):.3f} +/- {np.std(qualities):.3f}\")\n print(\"-------\")\n\nif __name__ == \"__main__\":\n compute_final_metrics(\"quality\", \"opt-profiling-data\", \"biodex-final-mab-k6-j4-budget150\")\n compute_final_metrics(\"cost\", \"min-cost-at-quality-data\", \"biodex-pareto-min-cost-budget150-k6-j4\")\n compute_final_metrics(\"latency\", \"min-latency-at-quality-data\", \"biodex-pareto-min-latency-budget150-k6-j4\")\n" }, { "path": "abacus-research/score_cuad.py", "content": "import json\nimport os\n\nimport numpy as np\n\n\ndef compute_final_metrics(metric: str, dir: str, exp_base_name: str):\n qualities = []\n opt_costs, run_costs = [], []\n opt_times, run_times = [], []\n total_costs, total_times = [], []\n print(f\"--- {metric} ---\")\n for seed in range(10):\n exp_name = f\"{exp_base_name}-seed{seed}\"\n if os.path.exists(f\"{dir}/{exp_name}-metrics.json\") is False:\n print(f\"Missing {dir}/{exp_name}-metrics.json\")\n continue\n with open(f\"{dir}/{exp_name}-metrics.json\") as f:\n metrics = json.load(f)\n qualities.append(metrics[\"f1\"])\n opt_costs.append(metrics[\"optimization_cost\"])\n opt_times.append(metrics[\"optimization_time\"])\n run_costs.append(metrics[\"plan_execution_cost\"])\n run_times.append(metrics[\"plan_execution_time\"])\n total_costs.append(metrics[\"total_execution_cost\"])\n total_times.append(metrics[\"total_execution_time\"])\n \n print(f\"Opt. Cost: {np.mean(opt_costs):.3f} +/- {np.std(opt_costs):.3f}\")\n print(f\"Opt. Time: {np.mean(opt_times):.3f} +/- {np.std(opt_times):.3f}\")\n print(f\"Run Cost: {np.mean(run_costs):.3f} +/- {np.std(run_costs):.3f}\")\n print(f\"Run Time: {np.mean(run_times):.3f} +/- {np.std(run_times):.3f}\")\n print(f\"Total Cost: {np.mean(total_costs):.3f} +/- {np.std(total_costs):.3f}\")\n print(f\"Total Time: {np.mean(total_times):.3f} +/- {np.std(total_times):.3f}\")\n print(f\"Quality: {np.mean(qualities):.3f} +/- {np.std(qualities):.3f}\")\n print(\"-------\")\n\nif __name__ == \"__main__\":\n compute_final_metrics(\"quality\", \"opt-profiling-data\", \"cuad-final-mab-k6-j4-budget50\")\n compute_final_metrics(\"cost\", \"opt-profiling-data\", \"cuad-final-mincost-k6-j4-budget50\")\n compute_final_metrics(\"latency\", \"opt-profiling-data\", \"cuad-final-minlatency-k6-j4-budget50\")" }, { "path": "abacus-research/score_mmqa.py", "content": "import json\n\nimport numpy as np\n\n\ndef compute_final_metrics(metric: str, dir: str, exp_base_name: str):\n qualities = []\n opt_costs, run_costs = [], []\n opt_times, run_times = [], []\n total_costs, total_times = [], []\n print(f\"--- {metric} ---\")\n for seed in range(10):\n exp_name = f\"{exp_base_name}-seed{seed}\"\n with open(f\"{dir}/{exp_name}-stats.json\") as f:\n metrics = json.load(f)\n qualities.append(metrics[\"f1\"])\n opt_costs.append(metrics[\"optimization_cost\"])\n opt_times.append(metrics[\"optimization_time\"])\n run_costs.append(metrics[\"plan_execution_cost\"])\n run_times.append(metrics[\"plan_execution_time\"])\n total_costs.append(metrics[\"total_execution_cost\"])\n total_times.append(metrics[\"total_execution_time\"])\n \n print(f\"Opt. Cost: {np.mean(opt_costs):.3f} +/- {np.std(opt_costs):.3f}\")\n print(f\"Opt. Time: {np.mean(opt_times):.3f} +/- {np.std(opt_times):.3f}\")\n print(f\"Run Cost: {np.mean(run_costs):.3f} +/- {np.std(run_costs):.3f}\")\n print(f\"Run Time: {np.mean(run_times):.3f} +/- {np.std(run_times):.3f}\")\n print(f\"Total Cost: {np.mean(total_costs):.3f} +/- {np.std(total_costs):.3f}\")\n print(f\"Total Time: {np.mean(total_times):.3f} +/- {np.std(total_times):.3f}\")\n print(f\"Quality: {np.mean(qualities):.3f} +/- {np.std(qualities):.3f}\")\n print(\"-------\")\n\nif __name__ == \"__main__\":\n compute_final_metrics(\"quality\", \"opt-profiling-data\", \"mmqa-final-mab-k6-j4-budget150\")\n compute_final_metrics(\"cost\", \"opt-profiling-data\", \"mmqa-final-mincost-k6-j4-budget150\")\n compute_final_metrics(\"latency\", \"opt-profiling-data\", \"mmqa-final-minlatency-k6-j4-budget150\")\n" }, { "path": "abacus-research/score_mmqa_complex.py", "content": "import json\nimport os\n\nimport numpy as np\n\n\ndef compute_final_metrics(metric: str, dir: str, exp_base_name: str):\n qualities = []\n opt_costs, run_costs = [], []\n opt_times, run_times = [], []\n total_costs, total_times = [], []\n print(f\"--- {metric} ---\")\n for seed in [0, 1, 2, 3, 4, 6, 8, 9]:\n exp_name = f\"{exp_base_name}-seed{seed}\"\n if os.path.exists(f\"{dir}/{exp_name}-stats.json\"):\n with open(f\"{dir}/{exp_name}-stats.json\") as f:\n metrics = json.load(f)\n qualities.append(metrics[\"f1\"])\n opt_costs.append(metrics[\"optimization_cost\"])\n opt_times.append(metrics[\"optimization_time\"])\n run_costs.append(metrics[\"plan_execution_cost\"])\n run_times.append(metrics[\"plan_execution_time\"])\n total_costs.append(metrics[\"total_execution_cost\"])\n total_times.append(metrics[\"total_execution_time\"])\n \n print(f\"Opt. Cost: {np.mean(opt_costs):.3f} +/- {np.std(opt_costs):.3f}\")\n print(f\"Opt. Time: {np.mean(opt_times):.3f} +/- {np.std(opt_times):.3f}\")\n print(f\"Run Cost: {np.mean(run_costs):.3f} +/- {np.std(run_costs):.3f}\")\n print(f\"Run Time: {np.mean(run_times):.3f} +/- {np.std(run_times):.3f}\")\n print(f\"Total Cost: {np.mean(total_costs):.3f} +/- {np.std(total_costs):.3f}\")\n print(f\"Total Time: {np.mean(total_times):.3f} +/- {np.std(total_times):.3f}\")\n print(f\"Quality: {np.mean(qualities):.3f} +/- {np.std(qualities):.3f}\")\n print(\"-------\")\n\nif __name__ == \"__main__\":\n compute_final_metrics(\"quality\", \"opt-profiling-data\", \"mmqa-complex-final-mab-k6-j4-budget350\")\n compute_final_metrics(\"cost\", \"opt-profiling-data\", \"mmqa-complex-mincost-k6-j4-budget350\")\n compute_final_metrics(\"latency\", \"opt-profiling-data\", \"mmqa-complex-minlatency-k6-j4-budget350\")\n" }, { "path": "abacus-research/setup_cuad_data.py", "content": "#!/usr/bin/env python\n\"\"\"\nScript to download CUAD dataset and set up local data directory.\nThis replaces the need for HuggingFace datasets library.\n\"\"\"\n\nimport os\nimport urllib.request\nimport zipfile\n\n\ndef setup_cuad_data():\n # Create cuad-data directory\n data_dir = \"cuad-data\"\n if not os.path.exists(data_dir):\n os.makedirs(data_dir)\n print(f\"Created directory: {data_dir}\")\n \n # Download CUAD data zip file\n data_url = \"https://github.com/TheAtticusProject/cuad/raw/main/data.zip\"\n zip_path = os.path.join(data_dir, \"data.zip\")\n \n if not os.path.exists(zip_path):\n print(f\"Downloading CUAD data from {data_url}...\")\n urllib.request.urlretrieve(data_url, zip_path)\n print(f\"Downloaded to {zip_path}\")\n else:\n print(f\"Data already downloaded at {zip_path}\")\n \n # Extract the zip file\n print(\"Extracting data...\")\n with zipfile.ZipFile(zip_path, 'r') as zip_ref:\n zip_ref.extractall(data_dir)\n print(f\"Extracted data to {data_dir}\")\n \n # Download the dataset loading script (for reference, not actually used)\n script_url = \"https://huggingface.co/datasets/theatticusproject/cuad-qa/resolve/main/cuad-qa.py\"\n script_path = os.path.join(data_dir, \"cuad-qa.py\")\n \n if not os.path.exists(script_path):\n print(f\"Downloading CUAD dataset script from {script_url}...\")\n urllib.request.urlretrieve(script_url, script_path)\n print(f\"Downloaded to {script_path}\")\n else:\n print(f\"Script already exists at {script_path}\")\n \n # List extracted files\n print(\"\\nExtracted files:\")\n for file in os.listdir(data_dir):\n if file.endswith('.json'):\n file_path = os.path.join(data_dir, file)\n size = os.path.getsize(file_path) / (1024 * 1024) # Size in MB\n print(f\" - {file} ({size:.2f} MB)\")\n \n print(\"\\nSetup complete! CUAD data is ready in the 'cuad-data' directory.\")\n print(\"\\nTo use this data in your scripts, update the data loading to:\")\n print(\" - train data: cuad-data/train_separate_questions.json\")\n print(\" - test data: cuad-data/test.json\")\n \n return data_dir\n\nif __name__ == \"__main__\":\n setup_cuad_data()" }, { "path": "demos/audio-demo.py", "content": "import os\n\nimport kagglehub\n\nimport palimpzest as pz\n\n\nclass SmallAudioDataset(pz.AudioFileDataset):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n\n # Limit to first 10 audio files for demo purposes\n self.filepaths = self.filepaths[:10]\n\n\nif __name__ == \"__main__\":\n # Download latest version\n path = kagglehub.dataset_download(\"rushibalajiputthewad/sound-classification-of-animal-voice\")\n print(f\"Dataset downloaded to: {path}\")\n\n # create simple plan to classify animal sounds\n plan = SmallAudioDataset(id=\"animal-sounds\", path=os.path.join(path, \"Animal-Soundprepros\"))\n plan = plan.sem_map(cols=[{\"name\": \"animal\", \"type\": str, \"description\": \"The type of animal making the sound in the recording.\"}])\n\n # run plan un-optimized\n config = pz.QueryProcessorConfig(\n policy=pz.MaxQuality(),\n # available_models=[pz.Model.GEMINI_2_0_FLASH, pz.Model.GEMINI_2_5_FLASH, pz.Model.GEMINI_2_5_PRO],\n )\n output = plan.run(config)\n\n print(output.to_df())\n" }, { "path": "demos/caching-demo.py", "content": "#!/usr/bin/env python3\n\"\"\"\nRealistic Demo showcasing prompt caching capabilities in Palimpzest.\n\nThis demo processes multiple employee travel requests against a comprehensive\nCorporate Travel Policy. The policy text (~2000 tokens) is included in the\nsystem prompt, creating a realistic scenario for prompt caching where a large\nstatic context is reused across multiple dynamic inputs.\n\nWorkload:\n- Context: A lengthy 10-page Corporate Travel & Expense Policy.\n- Input: Short email requests from employees.\n- Task: Analyze each request for policy compliance, identifying violations and reimbursable amounts.\n\nSupported caching providers:\n- OpenAI (GPT-4o, GPT-4o-mini): Automatic prefix caching\n- Anthropic (Claude 3.5 Sonnet/Haiku): Explicit cache_control markers\n- Gemini: Implicit caching\n\"\"\"\n\nimport argparse\nimport os\nimport time\nfrom typing import List\n\nfrom dotenv import load_dotenv\n\nimport palimpzest as pz\nfrom palimpzest.constants import Model\nfrom palimpzest.core.lib.schemas import TextFile\n\nload_dotenv()\n\n# =============================================================================\n# MOCK DATA: CORPORATE TRAVEL POLICY (Static Context > 1024 tokens)\n# =============================================================================\nCORPORATE_TRAVEL_POLICY = \"\"\"\nGLOBAL CORP TRAVEL & EXPENSE POLICY (v2024.1)\n\nSECTION 1: OVERVIEW AND PHILOSOPHY\nGlobal Corp expects employees to act responsibly and professionally when incurring and submitting costs. \nThe company will reimburse employees for reasonable and necessary expenses incurred during approved business travel. \nThis policy applies to all employees, contractors, and consultants.\n\nSECTION 2: AIR TRAVEL\n2.1 Booking Window: All domestic flights must be booked at least 14 days in advance. International flights must be booked 21 days in advance.\n2.2 Class of Service:\n - Economy Class: Required for all domestic flights under 6 hours.\n - Premium Economy: Allowed for domestic flights over 6 hours or international flights under 8 hours.\n - Business Class: Allowed for international flights exceeding 8 hours duration.\n - First Class: Strictly prohibited unless approved by the CEO.\n2.3 Ancillary Fees:\n - Checked Bags: Up to two bags reimbursed for trips > 3 days. One bag for trips <= 3 days.\n - Wi-Fi: Reimbursed only if business justification is provided (e.g., \"urgent client deadline\").\n - Seat Selection: Fees > $50 require VP approval.\n\nSECTION 3: LODGING\n3.1 Hotel Caps (Nightly Rates excluding taxes):\n - Tier 1 Cities (NY, London, Tokyo, SF, Zurich): $350 USD\n - Tier 2 Cities (Chicago, Paris, Berlin, Austin): $250 USD\n - All Other Locations: $175 USD\n3.2 Room Type: Standard single rooms only. Suites are prohibited.\n3.3 Laundry: Reasonable laundry expenses reimbursed for trips exceeding 5 consecutive nights.\n\nSECTION 4: MEALS AND ENTERTAINMENT\n4.1 Daily Meal Allowance (Per Diem):\n - Tier 1 Cities: $100/day\n - Tier 2 Cities: $75/day\n - Others: $60/day\n4.2 Client Entertainment:\n - Must include at least one current or prospective client.\n - Cap is $150 per person (including employees).\n - Names and affiliations of all attendees must be documented.\n4.3 Alcohol:\n - Reimbursable only with dinner.\n - Moderate consumption allowed (max 2 drinks per person).\n - \"Top Shelf\" liquors prohibited.\n\nSECTION 5: GROUND TRANSPORTATION\n5.1 Ride Share/Taxi: Preferred mode for travel between airport and hotel.\n5.2 Car Rentals:\n - Class: Intermediate/Mid-size or smaller.\n - Insurance: Decline CDW/LDW (covered by corporate policy).\n - Fuel: Pre-paid fuel options are prohibited; cars must be returned full.\n5.3 Rail: Economy/Standard class only. Acela Business Class permitted for Northeast Corridor travel.\n\nSECTION 6: MISCELLANEOUS\n6.1 Tipping:\n - Meals: 15-20%\n - Taxis: 10-15%\n - Bellhop: $1-2 per bag\n6.2 Non-Reimbursable Items:\n - Personal grooming/toiletries.\n - Fines (parking, speeding).\n - Airline club memberships.\n - In-room movies.\n - Lost luggage/property.\n\nSECTION 7: SUBMISSION PROCESS\nExpenses must be submitted within 30 days of trip completion. Receipts required for all expenses > $25.\n\"\"\"\n\n# =============================================================================\n# MOCK DATA: EMPLOYEE REQUESTS (Dynamic Inputs)\n# =============================================================================\nEMPLOYEE_REQUESTS = [\n # Request 1: Compliant\n \"\"\"Subject: Trip to London\n I booked a flight to London (8.5 hours) in Business Class for the client summit. \n Hotel is $320/night. Meal expenses were about $90/day. \n Receipts attached.\"\"\",\n # Request 2: Violation (Booking window & First Class)\n \"\"\"Subject: Urgent NY Trip\n I need to fly to New York tomorrow. Booked First Class because it was the only seat left.\n Hotel is the Ritz at $500/night. \n Also expensed $40 for in-flight Wi-Fi to finish the Q3 report.\"\"\",\n # Request 3: Violation (Car Rental & Alcohol)\n \"\"\"Subject: Austin Conference\n Rented a luxury SUV for the team in Austin. \n Dinner with the team (no clients) came to $800 ($200/person) including 3 bottles of wine.\n Hotel was $240/night.\"\"\",\n # Request 4: Compliant (Tier 2 City)\n \"\"\"Subject: Berlin Site Visit\n Flew Economy to Berlin. Hotel was $220/night.\n Took a taxi from TXL ($45 + $5 tip).\n Daily meals averaged $70.\"\"\",\n # Request 5: Violation (Misc items)\n \"\"\"Subject: Tokyo Tech Symposium\n Trip duration: 4 days. \n Expensed:\n - Flight (Premium Econ, 11 hours)\n - Hotel ($340/night)\n - Laundry service ($60)\n - Forgotten toothbrush replacement ($15)\n - Parking ticket ($50)\n \"\"\",\n]\n\n# Output Schema\nOUTPUT_SCHEMA = [\n {\"name\": \"status\", \"type\": str, \"desc\": \"One of: 'COMPLIANT', 'PARTIAL_VIOLATION', 'MAJOR_VIOLATION'\"},\n {\n \"name\": \"violations\",\n \"type\": str,\n \"desc\": \"A list of specific policy violations found, referencing the specific section numbers (e.g., 'Violation of Section 2.2'). If compliant, return 'None'.\",\n },\n {\n \"name\": \"reimbursable_summary\",\n \"type\": str,\n \"desc\": \"A concise summary of what should be reimbursed vs rejected based on the policy text.\",\n },\n {\n \"name\": \"flag_for_review\",\n \"type\": bool,\n \"desc\": \"True if the request requires manual review by a manager (e.g. for high amounts or ambiguous justifications).\",\n },\n]\n\nTASK_DESC = f\"\"\"\nYou are an AI auditor for Global Corp. Your job is to review employee travel expense descriptions against the Corporate Travel Policy.\nThe full policy text is provided below. \n\n{CORPORATE_TRAVEL_POLICY}\n\nAnalyze the input email and determine if the expenses adhere to the policy.\n\"\"\"\n\n\nclass TravelRequestDataset(pz.IterDataset):\n \"\"\"Custom dataset that provides travel requests as text records.\"\"\"\n\n def __init__(self, requests: List[str]):\n super().__init__(id=\"travel_requests\", schema=TextFile)\n self.requests = requests\n\n def __len__(self):\n return len(self.requests)\n\n def __getitem__(self, idx: int):\n return {\n \"filename\": f\"request_{idx + 1}.txt\",\n \"contents\": self.requests[idx],\n }\n\n\n# Model mapping (Same as original)\nMODEL_MAPPING = {\n \"gpt-4o\": Model.GPT_4o,\n \"gpt-4o-mini\": Model.GPT_4o_MINI,\n \"claude-4-0-sonnet\": Model.CLAUDE_4_SONNET,\n # \"claude-3-7-sonnet\": Model.CLAUDE_3_7_SONNET, # deprecated model testing\n \"claude-4-5-haiku\": Model.CLAUDE_4_5_HAIKU,\n \"gemini-2.5-flash\": Model.GOOGLE_GEMINI_2_5_FLASH,\n # \"deepseek-v3\": Model.DEEPSEEK_V3,\n}\n\n\ndef get_model_from_string(model_str: str) -> Model:\n if model_str.lower() in MODEL_MAPPING:\n return MODEL_MAPPING[model_str.lower()]\n for model in Model:\n if model.value.lower() == model_str.lower():\n return model\n raise ValueError(f\"Unknown model: {model_str}\")\n\n\ndef print_cache_stats(execution_stats):\n \"\"\"Print cache-related statistics from execution.\"\"\"\n print(\"\\n\" + \"=\" * 60)\n print(\" CACHE STATISTICS & COST ANALYSIS\")\n print(\"=\" * 60)\n\n # Token counts are now disjoint:\n # - input_text_tokens: regular (non-cached) input tokens\n # - cache_read_tokens: tokens read from cache (hits)\n # - cache_creation_tokens: tokens written to cache\n regular_input = execution_stats.input_text_tokens\n cache_read = execution_stats.cache_read_tokens\n cache_creation = execution_stats.cache_creation_tokens\n total_output = execution_stats.output_text_tokens\n total_embedding = execution_stats.embedding_input_tokens\n\n # Logical total = regular + cache read + cache creation\n logical_total_input = regular_input + cache_read + cache_creation\n\n print(f\"{'Metric':<35} | {'Count':<15}\")\n print(\"-\" * 55)\n print(f\"{'Logical Total Input Tokens':<35} | {logical_total_input:,}\")\n print(f\"{' - Regular Input (full rate)':<35} | {regular_input:,}\")\n print(f\"{' - Cache Read (discounted)':<35} | {cache_read:,}\")\n print(f\"{' - Cache Creation':<35} | {cache_creation:,}\")\n print(\"-\" * 55)\n print(f\"{'Total Output Tokens':<35} | {total_output:,}\")\n if total_embedding > 0:\n print(f\"{'Total Embedding Input Tokens':<35} | {total_embedding:,}\")\n print(\"-\" * 55)\n print(f\"{'Total Execution Cost':<35} | ${execution_stats.total_execution_cost:.6f}\")\n\n # Calculate and display cache hit rate\n # Hit rate = cache_read / (regular_input + cache_read)\n total_cacheable = regular_input + cache_read\n if total_cacheable > 0:\n hit_rate = (cache_read / total_cacheable) * 100\n print(f\"\\nCache Hit Rate: {hit_rate:.1f}%\")\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Demo showcasing prompt caching in Palimpzest\")\n parser.add_argument(\"--model\", type=str, default=\"gpt-4o-mini\", help=\"Model to use\")\n parser.add_argument(\"--num-records\", type=int, default=5, help=\"Number of requests to process\")\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"Enable verbose output\")\n parser.add_argument(\"--profile\", action=\"store_true\", help=\"Save profiling data\")\n\n args = parser.parse_args()\n model = get_model_from_string(args.model)\n\n # Validate env vars (Simplified for brevity)\n if model.is_provider_openai() and not os.getenv(\"OPENAI_API_KEY\"):\n print(\"ERROR: OPENAI_API_KEY not set\")\n return\n if model.is_provider_anthropic() and not os.getenv(\"ANTHROPIC_API_KEY\"):\n print(\"ERROR: ANTHROPIC_API_KEY not set\")\n return\n if (model.is_provider_google_ai_studio() or model.is_provider_vertex_ai()) and not os.getenv(\"GOOGLE_API_KEY\"):\n print(\"ERROR: GOOGLE_API_KEY not set\")\n return\n\n print(\"=\" * 60)\n print(\" PZ CACHING DEMO: CORPORATE AUDIT\")\n print(\"=\" * 60)\n print(f\"Model: {model.value}\")\n print(\n f\"Policy Context Size: ~{len(CORPORATE_TRAVEL_POLICY.split())} words (~{int(len(CORPORATE_TRAVEL_POLICY.split()) * 1.3)} tokens)\"\n )\n\n # Repeat the request list if user wants more records than we have mocks\n base_requests = EMPLOYEE_REQUESTS\n requests = []\n while len(requests) < args.num_records:\n requests.extend(base_requests)\n requests = requests[: args.num_records]\n\n print(f\"Processing {len(requests)} travel requests...\")\n\n # Build Plan\n dataset = TravelRequestDataset(requests)\n\n # The 'desc' field incorporates the huge CORPORATE_TRAVEL_POLICY string.\n # This ensures the System Prompt is large (>1024 tokens) and identical for all records.\n plan = dataset.sem_map(OUTPUT_SCHEMA, desc=TASK_DESC)\n\n config = pz.QueryProcessorConfig(\n policy=pz.MaxQuality(),\n verbose=args.verbose,\n execution_strategy=\"sequential\", # Sequential often easier to debug caching behavior initially\n available_models=[model],\n )\n\n start_time = time.time()\n result = plan.run(config)\n end_time = time.time()\n\n # Output Results\n print(\"\\n\" + \"=\" * 60)\n print(\" AUDIT RESULTS\")\n print(\"=\" * 60)\n for i, record in enumerate(result.data_records):\n print(f\"\\n[Request {i + 1}]\")\n print(f\"Status: {record.status}\")\n print(f\"Violations: {record.violations}\")\n print(f\"Summary: {record.reimbursable_summary}\")\n\n print_cache_stats(result.execution_stats)\n print(f\"\\nWall Clock Time: {end_time - start_time:.2f}s\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "demos/demo_core.py", "content": "#!/usr/bin/env python3\nimport json\nimport os\n\nimport pandas as pd\nfrom tabulate import tabulate\n\nimport palimpzest as pz\nfrom palimpzest.core.elements.groupbysig import GroupBySig\nfrom palimpzest.core.elements.records import DataRecord\n\nsci_paper_cols = [\n {\"name\": \"title\", \"type\": str, \"desc\": \"The title of the paper. This is a natural language title, not a number or letter.\"},\n {\"name\": \"publication_year\", \"type\": int, \"desc\": \"The year the paper was published. This is a number.\"},\n {\"name\": \"author\", \"type\": str, \"desc\": \"The name of the first author of the paper\"},\n {\"name\": \"institution\", \"type\": str, \"desc\": \"The institution of the first author of the paper\"},\n {\"name\": \"journal\", \"type\": str, \"desc\": \"The name of the journal the paper was published in\"},\n {\"name\": \"funding_agency\", \"type\": str, \"desc\": \"The name of the funding agency that supported the research\"},\n]\n\nemail_cols = [\n {\"name\": \"sender\", \"type\": str, \"desc\": \"The email address of the sender\"},\n {\"name\": \"subject\", \"type\": str, \"desc\": \"The subject of the email\"},\n]\n\ndog_image_cols = [\n {\"name\": \"breed\", \"type\": str, \"desc\": \"The breed of the dog\"},\n]\n\ndef build_sci_paper_plan(dataset):\n \"\"\"A dataset-independent declarative description of authors of good papers\"\"\"\n return pz.PDFFileDataset(id=\"science-papers\", path=dataset).sem_map(sci_paper_cols)\n\ndef build_test_pdf_plan(dataset):\n \"\"\"This tests whether we can process a PDF file\"\"\"\n return pz.PDFFileDataset(id=\"pdf-files\", path=dataset)\n\ndef build_mit_battery_paper_plan(dataset):\n \"\"\"A dataset-independent declarative description of authors of good papers\"\"\"\n sci_papers = pz.PDFFileDataset(id=\"science-papers\", path=dataset).sem_map(sci_paper_cols)\n battery_papers = sci_papers.sem_filter(\"The paper is about batteries\")\n mit_papers = battery_papers.sem_filter(\"The paper is from MIT\")\n return mit_papers\n\ndef build_enron_plan(dataset):\n \"\"\"Build a plan for processing Enron email data\"\"\"\n return pz.TextFileDataset(id=\"enron-emails\", path=dataset).sem_map(email_cols)\n\ndef compute_enron_stats(dataset):\n \"\"\"Compute statistics on Enron email data\"\"\"\n emails = pz.TextFileDataset(id=\"enron-emails\", path=dataset).sem_map(email_cols)\n subject_line_lengths = emails.sem_map([{\"name\": \"words\", \"type\": int, \"desc\": \"The number of words in the subject field\"}])\n return subject_line_lengths\n\ndef enron_gby_plan(dataset):\n \"\"\"Group Enron emails by sender\"\"\"\n emails = pz.TextFileDataset(id=\"enron-emails\", path=dataset).sem_map(email_cols)\n ops = [\"count\"]\n fields = [\"sender\"]\n groupbyfields = [\"sender\"]\n gby_desc = GroupBySig(groupbyfields, ops, fields)\n grouped_emails = emails.groupby(gby_desc)\n return grouped_emails\n\ndef enron_count_plan(dataset):\n \"\"\"Count total Enron emails\"\"\"\n emails = pz.TextFileDataset(id=\"enron-emails\", path=dataset).sem_map(email_cols)\n ops = [\"count\"]\n fields = [\"sender\"]\n groupbyfields = []\n gby_desc = GroupBySig(groupbyfields, ops, fields)\n count_emails = emails.groupby(gby_desc)\n return count_emails\n\ndef enron_average_count_plan(dataset):\n \"\"\"Calculate average number of emails per sender\"\"\"\n emails = pz.TextFileDataset(id=\"enron-emails\", path=dataset).sem_map(email_cols)\n ops = [\"count\"]\n fields = [\"sender\"]\n groupbyfields = [\"sender\"]\n gby_desc = GroupBySig(groupbyfields, ops, fields)\n grouped_emails = emails.groupby(gby_desc)\n ops = [\"average\"]\n fields = [\"count(sender)\"]\n groupbyfields = []\n gby_desc = GroupBySig(groupbyfields, ops, fields)\n average_emails_per_sender = grouped_emails.groupby(gby_desc)\n return average_emails_per_sender\n\ndef enron_limit_plan(dataset, limit=5):\n \"\"\"Get limited number of Enron emails\"\"\"\n emails = pz.TextFileDataset(id=\"enron-emails\", path=dataset).sem_map(email_cols)\n limit_data = emails.limit(limit)\n return limit_data\n\ndef build_image_plan(dataset):\n \"\"\"Build a plan for processing dog images\"\"\"\n images = pz.ImageFileDataset(id=\"dog-images\", path=dataset)\n filtered_images = images.sem_filter(\"The image contains one or more dogs\")\n dog_images = filtered_images.sem_map(dog_image_cols)\n return dog_images\n\ndef build_image_agg_plan(dataset):\n \"\"\"Build a plan for aggregating dog images by breed\"\"\"\n images = pz.ImageFileDataset(id=\"dog-images\", path=dataset)\n filtered_images = images.sem_filter(\"The image contains one or more dogs\")\n dog_images = filtered_images.sem_map(dog_image_cols)\n ops = [\"count\"]\n fields = [\"breed\"]\n groupbyfields = [\"breed\"]\n gby_desc = GroupBySig(groupbyfields, ops, fields)\n grouped_dog_images = dog_images.groupby(gby_desc)\n return grouped_dog_images\n\ndef build_join_plan(dataset1, dataset2):\n \"\"\"Build a plan that joins two datasets\"\"\"\n ds1 = pz.TextFileDataset(id=\"enron-emails\", path=dataset1).sem_map(email_cols)\n ds2 = pz.TextFileDataset(id=\"other-enron-emails\", path=dataset2).sem_map(email_cols)\n joined = ds1.sem_join(ds2, condition=\"sender\")\n return joined\n\ndef build_join_image_plan(dataset1, dataset2):\n \"\"\"Build a plan that joins two datasets with images\"\"\"\n ds1 = pz.ImageFileDataset(id=\"dog-images\", path=dataset1).sem_map(dog_image_cols)\n ds2 = pz.ImageFileDataset(id=\"other-dog-images\", path=dataset2).sem_map(dog_image_cols)\n joined = ds1.sem_join(ds2, condition=\"breed\")\n return joined\n\ndef get_task_config(task, dataset, join_dataset=None):\n \"\"\"Get configuration for a specific task\"\"\"\n if task == \"paper\":\n root_set = build_mit_battery_paper_plan(dataset)\n cols = [\"title\", \"publication_year\", \"author\", \"institution\", \"journal\", \"funding_agency\"]\n stat_path = \"profiling-data/paper-profiling.json\"\n elif task == \"enron\":\n root_set = build_enron_plan(dataset)\n cols = [\"sender\", \"subject\"]\n stat_path = \"profiling-data/enron-profiling.json\"\n elif task == \"enronGby\":\n root_set = enron_gby_plan(dataset)\n cols = [\"sender\", \"count(sender)\"]\n stat_path = \"profiling-data/egby-profiling.json\"\n elif task in (\"enronCount\", \"count\"):\n root_set = enron_count_plan(dataset)\n cols = [\"count(sender)\"]\n stat_path = \"profiling-data/ecount-profiling.json\"\n elif task in (\"enronAvgCount\", \"average\"):\n root_set = enron_average_count_plan(dataset)\n cols = [\"average(count(sender))\"]\n stat_path = \"profiling-data/e-profiling.json\"\n elif task == \"enronmap\":\n root_set = compute_enron_stats(dataset)\n cols = [\"sender\", \"subject\", \"value\"]\n stat_path = \"profiling-data/emap-profiling.json\"\n elif task == \"pdftest\":\n root_set = build_test_pdf_plan(dataset)\n cols = [\"filename\"]\n stat_path = \"profiling-data/pdftest-profiling.json\"\n elif task == \"scitest\":\n root_set = build_sci_paper_plan(dataset)\n cols = [\"title\", \"author\", \"institution\", \"journal\", \"funding_agency\"]\n stat_path = \"profiling-data/scitest-profiling.json\"\n elif task == \"image\":\n root_set = build_image_plan(dataset)\n cols = None\n stat_path = \"profiling-data/image-profiling.json\"\n elif task == \"gbyImage\":\n root_set = build_image_agg_plan(dataset)\n cols = [\"breed\", \"count(breed)\"]\n stat_path = \"profiling-data/gbyImage-profiling.json\"\n elif task == \"limit\":\n root_set = enron_limit_plan(dataset, 5)\n cols = [\"sender\", \"subject\"]\n stat_path = \"profiling-data/limit-profiling.json\"\n elif task == \"join\":\n root_set = build_join_plan(dataset, join_dataset)\n cols = [\"filename\", \"sender\", \"subject\"]\n stat_path = \"profiling-data/join-profiling.json\"\n elif task == \"joinImage\":\n root_set = build_join_image_plan(dataset, join_dataset)\n cols = None\n stat_path = \"profiling-data/joinImage-profiling.json\"\n else:\n raise ValueError(f\"Unknown task: {task}\")\n \n return root_set, cols, stat_path\n\ndef execute_task(task, dataset, policy, join_dataset=None, verbose=False, profile=False, execution_strategy=\"sequential\", optimizer_strategy=\"pareto\"):\n \"\"\"Execute a task and return results\"\"\"\n root_set, cols, stat_path = get_task_config(task, dataset, join_dataset)\n config = pz.QueryProcessorConfig(\n policy=policy,\n verbose=verbose,\n execution_strategy=execution_strategy,\n optimizer_strategy=optimizer_strategy,\n )\n data_record_collection = root_set.run(config)\n\n if profile:\n os.makedirs(\"profiling-data\", exist_ok=True)\n with open(stat_path, \"w\") as f:\n json.dump(data_record_collection.execution_stats.to_json(), f)\n\n return data_record_collection.data_records, data_record_collection.execution_stats, cols\n\ndef format_results_table(records: list[DataRecord], cols=None):\n \"\"\"Format records as a table\"\"\"\n records = [record.to_dict(include_bytes=False) for record in records]\n records_df = pd.DataFrame(records)\n print_cols = records_df.columns if cols is None else cols\n final_df = records_df[print_cols] if not records_df.empty else pd.DataFrame(columns=print_cols)\n return tabulate(final_df, headers=\"keys\", tablefmt=\"psql\")\n" }, { "path": "demos/enron-demo.py", "content": "import json\nimport os\n\nimport palimpzest as pz\nfrom palimpzest.core.lib.schemas import TextFile\n\n\nclass EnronValidator(pz.Validator):\n def __init__(self, labels_file: str):\n super().__init__()\n\n self.filename_to_labels = {}\n if labels_file:\n with open(labels_file) as f:\n self.filename_to_labels = json.load(f)\n\n def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n filename = input_record[\"filename\"]\n labels = self.filename_to_labels[filename]\n if len(labels) == 0:\n return None\n\n labels = labels[0]\n return (float(labels[\"sender\"] == output[\"sender\"]) + float(labels[\"subject\"] == output[\"subject\"])) / 2.0\n\n\nclass EnronDataset(pz.IterDataset):\n def __init__(self, dir: str, labels_file: str | None = None, split: str = \"test\"):\n super().__init__(id=\"enron\", schema=TextFile)\n self.filepaths = [os.path.join(dir, filename) for filename in os.listdir(dir)]\n self.filepaths = self.filepaths[:50] if split == \"train\" else self.filepaths[50:150]\n self.filename_to_labels = {}\n if labels_file:\n with open(labels_file) as f:\n self.filename_to_labels = json.load(f)\n\n def __len__(self):\n return len(self.filepaths)\n\n def __getitem__(self, idx: int):\n # get input fields\n filepath = self.filepaths[idx]\n filename = os.path.basename(filepath)\n with open(filepath) as f:\n contents = f.read()\n\n # create item with fields\n item = {\"filename\": filename, \"contents\": contents}\n\n return item\n\n\nif __name__ == \"__main__\":\n # create validator and train_dataset\n validator = EnronValidator(labels_file=\"testdata/enron-eval-medium-labels.json\")\n train_dataset = EnronDataset(dir=\"testdata/enron-eval-medium\", split=\"train\")\n\n # construct plan\n plan = EnronDataset(dir=\"testdata/enron-eval-medium\", split=\"test\")\n plan = plan.sem_map([\n {\"name\": \"subject\", \"type\": str, \"desc\": \"The subject of the email\"},\n {\"name\": \"sender\", \"type\": str, \"desc\": \"The email address of the email's sender\"},\n ])\n plan = plan.sem_filter(\n 'The email refers to a fraudulent scheme (i.e., \"Raptor\", \"Deathstar\", \"Chewco\", and/or \"Fat Boy\")',\n depends_on=[\"contents\"],\n )\n plan = plan.sem_filter(\n \"The email is not quoting from a news article or an article written by someone outside of Enron\",\n depends_on=[\"contents\"],\n )\n\n # execute pz plan\n config = pz.QueryProcessorConfig(\n policy=pz.MaxQuality(),\n execution_strategy=\"parallel\",\n k=5,\n j=6,\n sample_budget=100,\n max_workers=20,\n progress=True,\n )\n output = plan.optimize_and_run(train_dataset=train_dataset, validator=validator, config=config)\n\n # print output dataframe\n print(output.to_df())\n\n # print precision and recall\n with open(\"testdata/enron-eval-medium-labels.json\") as f:\n filename_to_labels = json.load(f)\n test_filenames = os.listdir(\"testdata/enron-eval-medium\")[50:150]\n filename_to_labels = {k: v for k, v in filename_to_labels.items() if k in test_filenames}\n\n target_filenames = set(filename for filename, labels in filename_to_labels.items() if labels != [])\n pred_filenames = set(output.to_df()[\"filename\"])\n tp = sum(filename in target_filenames for filename in pred_filenames)\n fp = len(pred_filenames) - tp\n fn = len(target_filenames) - tp\n\n print(f\"PRECISION: {tp/(tp + fp) if tp + fp > 0 else 0.0:.3f}\")\n print(f\"RECALL: {tp/(tp + fn) if tp + fn > 0 else 0.0:.3f}\")\n" }, { "path": "demos/image-demo.py", "content": "#!/usr/bin/env python3\n\"\"\"This scripts is a demo for image processing, it is simply an abridged version of simpleDemo.py\"\"\"\n\nimport os\nimport time\n\nimport gradio as gr\nimport numpy as np\nfrom PIL import Image\n\nimport palimpzest as pz\n\nif not os.environ.get(\"OPENAI_API_KEY\"):\n from palimpzest.utils.env_helpers import load_env\n\n load_env()\n\ndog_image_cols = [\n {\"name\": \"breed\", \"type\": str, \"desc\": \"The breed of the dog\"},\n]\n\ndef build_image_plan(dataset):\n images = pz.ImageFileDataset(id=\"images\", path=dataset)\n filtered_images = images.sem_filter(\"The image contains one or more dogs\")\n dog_images = filtered_images.sem_map(dog_image_cols)\n return dog_images\n\n\nif __name__ == \"__main__\":\n # parse arguments\n start_time = time.time()\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n print(\"Starting image task\")\n policy = pz.MaxQuality()\n plan = build_image_plan(\"testdata/images-tiny\")\n config = pz.QueryProcessorConfig(policy=policy)\n data_record_collection = plan.run(config)\n\n imgs, breeds = [], []\n for record in data_record_collection:\n print(\"Trying to open \", record.filename)\n path = os.path.join(\"testdata/images-tiny/\", record.filename)\n img = Image.open(path).resize((128, 128))\n img_arr = np.asarray(img)\n imgs.append(img_arr)\n breeds.append(record.breed)\n\n with gr.Blocks() as demo:\n img_blocks, breed_blocks = [], []\n for img, breed in zip(imgs, breeds):\n with gr.Row():\n with gr.Column():\n img_blocks.append(gr.Image(value=img))\n with gr.Column():\n breed_blocks.append(gr.Textbox(value=breed))\n\n plan_str = list(data_record_collection.execution_stats.plan_strs.values())[0]\n gr.Textbox(value=plan_str, info=\"Query Plan\")\n\n end_time = time.time()\n print(\"Elapsed time:\", end_time - start_time)\n demo.launch()\n" }, { "path": "demos/join-data/animal-texts/animal1.txt", "content": "The quick red fox jumped over the fence.\n" }, { "path": "demos/join-data/animal-texts/animal2.txt", "content": "The black dog sat next to the bed.\n" }, { "path": "demos/join-data/animal-texts/animal3.txt", "content": "The white polar bear swam gently in the ocean.\n" }, { "path": "demos/join-data/animal-texts/animal4.txt", "content": "The labrador swam in the lake, the sun glistening off its shiny black coat.\n" }, { "path": "demos/join-data/animal-texts/animal5.txt", "content": "Clifford was a big red dog.\n" }, { "path": "demos/join-data/animal-texts/animal6.txt", "content": "The elephant was wise and grey.\n" }, { "path": "demos/join-demo.py", "content": "import argparse\n\nimport palimpzest as pz\n\n# define columns for datasets\ntext_animal_cols = [\n {\"name\": \"animal\", \"type\": str, \"desc\": \"The type of animal mentioned in the text\"},\n {\"name\": \"color\", \"type\": str, \"desc\": \"The color of the animal mentioned in the text\"},\n]\nimage_animal_cols = [\n {\"name\": \"animal\", \"type\": str, \"desc\": \"The type of animal in the image\"},\n {\"name\": \"color\", \"type\": str, \"desc\": \"The color of the animal in the image\"},\n]\n\n# query plans\ndef run_text_join():\n \"\"\"Build a plan that joins two datasets\"\"\"\n ds1 = pz.TextFileDataset(id=\"animals1\", path=\"join-data/animal-texts/\").sem_map(text_animal_cols)\n ds2 = pz.TextFileDataset(id=\"animals2\", path=\"join-data/animal-texts/\").sem_map(text_animal_cols)\n ds3 = ds1.sem_join(ds2, condition=\"both animals are canines with the same color\")\n config = pz.QueryProcessorConfig(\n policy=pz.MaxQuality(),\n execution_strategy=\"parallel\",\n join_parallelism=64,\n )\n data_record_collection = ds3.run(config)\n print(data_record_collection.to_df())\n\n\ndef run_image_join():\n \"\"\"Build a plan that joins two datasets with images\"\"\"\n ds1 = pz.ImageFileDataset(id=\"animals1\", path=\"join-data/animal-images/\").sem_map(image_animal_cols)\n ds2 = pz.ImageFileDataset(id=\"animals2\", path=\"join-data/animal-images/\").sem_map(image_animal_cols)\n ds3 = ds1.sem_join(ds2, condition=\"both animals are canines with the same color\")\n config = pz.QueryProcessorConfig(\n policy=pz.MaxQuality(),\n execution_strategy=\"parallel\",\n join_parallelism=64,\n )\n data_record_collection = ds3.run(config)\n print(data_record_collection.to_df())\n\n\ndef run_text_image_join():\n \"\"\"Build a plan that joins a dataset with text to a dataset with images\"\"\"\n ds1 = pz.TextFileDataset(id=\"animals1\", path=\"join-data/animal-texts/\").sem_map(text_animal_cols)\n ds2 = pz.ImageFileDataset(id=\"animals2\", path=\"join-data/animal-images/\").sem_map(image_animal_cols)\n ds3 = ds1.sem_join(ds2, condition=\"both animals are canines with the same color\")\n config = pz.QueryProcessorConfig(\n policy=pz.MaxQuality(),\n execution_strategy=\"parallel\",\n join_parallelism=64,\n )\n data_record_collection = ds3.run(config)\n print(data_record_collection.to_df())\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser(description=\"Run the Palimpzest join demo.\")\n parser.add_argument(\"--task\", type=str, help=\"Which join demo to run\")\n args = parser.parse_args()\n\n if args.task == \"text-join\":\n run_text_join()\n elif args.task == \"image-join\":\n run_image_join()\n elif args.task == \"text-image-join\":\n run_text_image_join()\n else:\n print(\"Please provide a valid task: one of 'text-join', 'image-join', 'text-image-join'\")\n exit(1)\n" }, { "path": "demos/paper-demo.py", "content": "import argparse\nimport json\nimport os\n\nimport gradio as gr\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\n\nimport palimpzest as pz\nfrom palimpzest.core.lib.schemas import ImageFilepath\nfrom palimpzest.utils.udfs import xls_to_tables\n\n\ndef print_table(records, cols=None, plan_str=None):\n \"\"\"Helper function to print execution results using Gradio\"\"\"\n if len(records) == 0:\n print(\"No records met search criteria\")\n return\n\n records = [record.to_dict() for record in records]\n records_df = pd.DataFrame(records)\n print_cols = records_df.columns if cols is None else cols\n\n with gr.Blocks() as demo:\n gr.Dataframe(records_df[print_cols])\n\n if plan_str is not None:\n gr.Textbox(value=plan_str, info=\"Physical Plan\")\n\n demo.launch()\n\n\n# Addresses far from MIT; we use a simple lookup like this to make the\n# experiments re-producible w/out needed a Google API key for geocoding lookups\nFAR_AWAY_ADDRS = [\n \"Melcher St\",\n \"Sleeper St\",\n \"437 D St\",\n \"Seaport Blvd\",\n \"50 Liberty Dr\",\n \"Telegraph St\",\n \"Columbia Rd\",\n \"E 6th St\",\n \"E 7th St\",\n \"E 5th St\",\n]\n\n\ndef within_two_miles_of_mit(record: dict):\n # NOTE: I'm using this hard-coded function so that folks w/out a\n # Geocoding API key from google can still run this example\n try:\n return not any([street.lower() in record[\"address\"].lower() for street in FAR_AWAY_ADDRS])\n except Exception:\n return False\n\n\ndef in_price_range(record: dict):\n try:\n price = record[\"price\"]\n if isinstance(price, str):\n price = price.strip()\n price = int(price.replace(\"$\", \"\").replace(\",\", \"\"))\n return 6e5 < price <= 2e6\n except Exception:\n return False\n\nemail_cols = [\n {\"name\": \"sender\", \"type\": str, \"desc\": \"The email address of the sender\"},\n {\"name\": \"subject\", \"type\": str, \"desc\": \"The subject of the email\"},\n]\n\ncase_data_cols = [\n {\"name\": \"case_submitter_id\", \"type\": str, \"desc\": \"The ID of the case\"},\n {\"name\": \"age_at_diagnosis\", \"type\": int | float, \"desc\": \"The age of the patient at the time of diagnosis\"},\n {\"name\": \"race\", \"type\": str, \"desc\": \"An arbitrary classification of a taxonomic group that is a division of a species.\"},\n {\"name\": \"ethnicity\", \"type\": str, \"desc\": \"Whether an individual describes themselves as Hispanic or Latino or not.\"},\n {\"name\": \"gender\", \"type\": str, \"desc\": \"Text designations that identify gender.\"},\n {\"name\": \"vital_status\", \"type\": str, \"desc\": \"The vital status of the patient\"},\n {\"name\": \"ajcc_pathologic_t\", \"type\": str, \"desc\": \"Code of pathological T (primary tumor) to define the size or contiguous extension of the primary tumor (T), using staging criteria from the American Joint Committee on Cancer (AJCC).\"},\n {\"name\": \"ajcc_pathologic_n\", \"type\": str, \"desc\": \"The codes that represent the stage of cancer based on the nodes present (N stage) according to criteria based on multiple editions of the AJCC's Cancer Staging Manual.\"},\n {\"name\": \"ajcc_pathologic_stage\", \"type\": str, \"desc\": \"The extent of a cancer, especially whether the disease has spread from the original site to other parts of the body based on AJCC staging criteria.\"},\n {\"name\": \"tumor_grade\", \"type\": int | float, \"desc\": \"Numeric value to express the degree of abnormality of cancer cells, a measure of differentiation and aggressiveness.\"},\n {\"name\": \"tumor_focality\", \"type\": str, \"desc\": \"The text term used to describe whether the patient's disease originated in a single location or multiple locations.\"},\n {\"name\": \"tumor_largest_dimension_diameter\", \"type\": int | float, \"desc\": \"The tumor largest dimension diameter.\"},\n {\"name\": \"primary_diagnosis\", \"type\": str, \"desc\": \"Text term used to describe the patient's histologic diagnosis, as described by the World Health Organization's (WHO) International Classification of Diseases for Oncology (ICD-O).\"},\n {\"name\": \"morphology\", \"type\": str, \"desc\": \"The Morphological code of the tumor, as described by the World Health Organization's (WHO) International Classification of Diseases for Oncology (ICD-O).\"},\n {\"name\": \"tissue_or_organ_of_origin\", \"type\": str, \"desc\": \"The text term used to describe the anatomic site of origin, of the patient's malignant disease, as described by the World Health Organization's (WHO) International Classification of Diseases for Oncology (ICD-O).\"},\n {\"name\": \"study\", \"type\": str, \"desc\": \"The last name of the author of the study, from the table name\"},\n {\"name\": \"filename\", \"type\": str, \"desc\": \"The name of the file the record was extracted from\"}\n]\n\nreal_estate_listing_cols = [\n {\"name\": \"listing\", \"type\": str, \"desc\": \"The name of the listing\"},\n {\"name\": \"text_content\", \"type\": str, \"desc\": \"The content of the listing's text description\"},\n {\"name\": \"image_filepaths\", \"type\": list[ImageFilepath], \"desc\": \"A list of the filepaths for each image of the listing\"},\n]\n\nreal_estate_text_cols = [\n {\"name\": \"address\", \"type\": str, \"desc\": \"The address of the property\"},\n {\"name\": \"price\", \"type\": int | float, \"desc\": \"The listed price of the property\"},\n]\n\nreal_estate_image_cols = [\n {\"name\": \"is_modern_and_attractive\", \"type\": bool, \"desc\": \"True if the home interior design is modern and attractive and False otherwise\"},\n {\"name\": \"has_natural_sunlight\", \"type\": bool, \"desc\": \"True if the home interior has lots of natural sunlight and False otherwise\"},\n]\n\ntable_cols = [\n {\"name\": \"rows\", \"type\": list[str], \"desc\": \"The rows of the table\"},\n {\"name\": \"header\", \"type\": list[str], \"desc\": \"The header of the table\"},\n {\"name\": \"name\", \"type\": str, \"desc\": \"The name of the table\"},\n {\"name\": \"filename\", \"type\": str, \"desc\": \"The name of the file the table was extracted from\"}\n]\n\n\nclass RealEstateListingDataset(pz.IterDataset):\n def __init__(self, listings_dir):\n super().__init__(id=\"real-estate\", schema=real_estate_listing_cols)\n self.listings_dir = listings_dir\n self.listings = sorted(os.listdir(self.listings_dir))\n self.listings = [file for file in self.listings if not file.startswith(\".\")]\n\n def __len__(self):\n return len(self.listings)\n\n def __getitem__(self, idx: int):\n # get listing\n listing = self.listings[idx]\n\n # get fields\n image_filepaths, text_content = [], None\n listing_dir = os.path.join(self.listings_dir, listing)\n for file in os.listdir(listing_dir):\n if file.endswith(\".txt\"):\n with open(os.path.join(listing_dir, file), \"rb\") as f:\n text_content = f.read().decode(\"utf-8\")\n elif file.endswith(\".png\"):\n image_filepaths.append(os.path.join(listing_dir, file))\n\n # construct and return dictionary with fields\n return {\"listing\": listing, \"text_content\": text_content, \"image_filepaths\": image_filepaths}\n\n\nif __name__ == \"__main__\":\n # parse arguments\n parser = argparse.ArgumentParser(description=\"Run a simple demo\")\n parser.add_argument(\"--viz\", default=False, action=\"store_true\", help=\"Visualize output in Gradio\")\n parser.add_argument(\"--verbose\", default=False, action=\"store_true\", help=\"Print verbose output\")\n parser.add_argument(\"--profile\", default=False, action=\"store_true\", help=\"Profile execution\")\n parser.add_argument(\"--dataset\", type=str, help=\"The path to the dataset\")\n parser.add_argument(\n \"--workload\", type=str, help=\"The workload to run. One of enron, real-estate, medical-schema-matching.\"\n )\n parser.add_argument(\n \"--executor\",\n type=str,\n help=\"The plan executor to use. One of sequential, pipelined, parallel\",\n default=\"parallel\",\n )\n parser.add_argument(\n \"--policy\",\n type=str,\n help=\"One of 'mincost', 'mintime', 'maxquality'\",\n default=\"maxquality\",\n )\n\n args = parser.parse_args()\n\n # The user has to indicate the dataset id and the workload\n if args.dataset is None:\n print(\"Please provide a dataset id\")\n exit(1)\n if args.workload is None:\n print(\"Please provide a workload\")\n exit(1)\n\n # create directory for profiling data\n if args.profile:\n os.makedirs(\"profiling-data\", exist_ok=True)\n\n dataset = args.dataset\n workload = args.workload\n visualize = args.viz\n verbose = args.verbose\n profile = args.profile\n policy = pz.MaxQuality()\n if args.policy == \"mincost\":\n policy = pz.MinCost()\n elif args.policy == \"mintime\":\n policy = pz.MinTime()\n elif args.policy == \"maxquality\":\n policy = pz.MaxQuality()\n else:\n print(\"Policy not supported for this demo\")\n exit(1)\n\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n # create pz plan\n if workload == \"enron\":\n plan = pz.TextFileDataset(id=\"enron\", path=dataset)\n plan = plan.sem_map(email_cols)\n plan = plan.sem_filter(\n \"The email is not quoting from a news article or an article written by someone outside of Enron\",\n depends_on=[\"contents\"],\n )\n plan = plan.sem_filter(\n 'The email refers to a fraudulent scheme (i.e., \"Raptor\", \"Deathstar\", \"Chewco\", and/or \"Fat Boy\")',\n depends_on=[\"contents\"],\n )\n\n elif workload == \"real-estate\":\n plan = RealEstateListingDataset(dataset)\n plan = plan.sem_map(real_estate_text_cols, depends_on=\"text_content\")\n plan = plan.sem_map(real_estate_image_cols, depends_on=\"image_filepaths\")\n plan = plan.sem_filter(\n \"The interior is modern and attractive, and has lots of natural sunlight\",\n depends_on=[\"is_modern_and_attractive\", \"has_natural_sunlight\"],\n )\n plan = plan.filter(within_two_miles_of_mit, depends_on=\"address\")\n plan = plan.filter(in_price_range, depends_on=\"price\")\n\n elif workload == \"medical-schema-matching\":\n plan = dataset.add_columns(xls_to_tables, cols=table_cols, cardinality=pz.Cardinality.ONE_TO_MANY)\n plan = plan.sem_filter(\"The rows of the table contain the patient age\")\n plan = plan.sem_map(case_data_cols, cardinality=pz.Cardinality.ONE_TO_MANY)\n\n # construct config and run plan\n config = pz.QueryProcessorConfig(\n verbose=verbose,\n policy=policy,\n execution_strategy=args.executor,\n )\n data_record_collection = plan.run(config)\n print(data_record_collection.to_df())\n\n # save statistics\n if profile:\n stats_path = f\"profiling-data/{workload}-profiling.json\"\n execution_stats_dict = data_record_collection.execution_stats.to_json()\n with open(stats_path, \"w\") as f:\n json.dump(execution_stats_dict, f)\n\n # visualize output in Gradio\n if visualize:\n plan_str = list(data_record_collection.execution_stats.plan_strs.values())[-1]\n if workload == \"enron\":\n print_table(data_record_collection.data_records, cols=[\"sender\", \"subject\"], plan_str=plan_str)\n\n elif workload == \"real-estate\":\n fst_imgs, snd_imgs, thrd_imgs, addrs, prices = [], [], [], [], []\n for record in data_record_collection:\n addrs.append(record.address)\n prices.append(record.price)\n for idx, img_name in enumerate([\"img1.png\", \"img2.png\", \"img3.png\"]):\n path = os.path.join(dataset, record.listing, img_name)\n img = Image.open(path)\n img_arr = np.asarray(img)\n if idx == 0:\n fst_imgs.append(img_arr)\n elif idx == 1:\n snd_imgs.append(img_arr)\n elif idx == 2:\n thrd_imgs.append(img_arr)\n\n with gr.Blocks() as demo:\n fst_img_blocks, snd_img_blocks, thrd_img_blocks, addr_blocks, price_blocks = [], [], [], [], []\n for fst_img, snd_img, thrd_img, addr, price in zip(fst_imgs, snd_imgs, thrd_imgs, addrs, prices):\n with gr.Row(equal_height=True):\n with gr.Column():\n fst_img_blocks.append(gr.Image(value=fst_img))\n with gr.Column():\n snd_img_blocks.append(gr.Image(value=snd_img))\n with gr.Column():\n thrd_img_blocks.append(gr.Image(value=thrd_img))\n with gr.Row():\n with gr.Column():\n addr_blocks.append(gr.Textbox(value=addr, info=\"Address\"))\n with gr.Column():\n price_blocks.append(gr.Textbox(value=price, info=\"Price\"))\n\n plan_str = list(data_record_collection.execution_stats.plan_strs.values())[0]\n gr.Textbox(value=plan_str, info=\"Query Plan\")\n\n demo.launch()\n" }, { "path": "demos/real-estate-demo.py", "content": "import argparse\nimport os\n\nimport gradio as gr\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\n\nimport palimpzest as pz\nfrom palimpzest.core.lib.schemas import ImageFilepath\n\n\ndef print_table(records, cols=None, plan_str=None):\n \"\"\"Helper function to print execution results using Gradio\"\"\"\n if len(records) == 0:\n print(\"No records met search criteria\")\n return\n\n records = [record.to_dict() for record in records]\n records_df = pd.DataFrame(records)\n print_cols = records_df.columns if cols is None else cols\n\n with gr.Blocks() as demo:\n gr.Dataframe(records_df[print_cols])\n\n if plan_str is not None:\n gr.Textbox(value=plan_str, info=\"Physical Plan\")\n\n demo.launch()\n\n\n# Addresses far from MIT; we use a simple lookup like this to make the\n# experiments re-producible w/out needed a Google API key for geocoding lookups\nFAR_AWAY_ADDRS = [\n \"Melcher St\",\n \"Sleeper St\",\n \"437 D St\",\n \"Seaport Blvd\",\n \"50 Liberty Dr\",\n \"Telegraph St\",\n \"Columbia Rd\",\n \"E 6th St\",\n \"E 7th St\",\n \"E 5th St\",\n]\n\n\ndef within_two_miles_of_mit(record: dict):\n # NOTE: I'm using this hard-coded function so that folks w/out a\n # Geocoding API key from google can still run this example\n try:\n return not any([street.lower() in record[\"address\"].lower() for street in FAR_AWAY_ADDRS])\n except Exception:\n return False\n\n\ndef in_price_range(record: dict):\n try:\n price = record[\"price\"]\n if isinstance(price, str):\n price = price.strip()\n price = int(price.replace(\"$\", \"\").replace(\",\", \"\"))\n return 6e5 < price <= 2e6\n except Exception:\n return False\n\nreal_estate_listing_cols = [\n {\"name\": \"listing\", \"type\": str, \"desc\": \"The name of the listing\"},\n {\"name\": \"text_content\", \"type\": str, \"desc\": \"The content of the listing's text description\"},\n {\"name\": \"image_filepaths\", \"type\": list[ImageFilepath], \"desc\": \"A list of the filepaths for each image of the listing\"},\n]\n\nreal_estate_text_cols = [\n {\"name\": \"address\", \"type\": str, \"desc\": \"The address of the property\"},\n {\"name\": \"price\", \"type\": int | float, \"desc\": \"The listed price of the property\"},\n]\n\nreal_estate_image_cols = [\n {\"name\": \"is_modern_and_attractive\", \"type\": bool, \"desc\": \"True if the home interior design is modern and attractive and False otherwise\"},\n {\"name\": \"has_natural_sunlight\", \"type\": bool, \"desc\": \"True if the home interior has lots of natural sunlight and False otherwise\"},\n]\n\n# class RealEstateValidator(pz.Validator):\n# def __init__(self, labels_file: str):\n# super().__init__()\n# with open(labels_file) as f:\n# self.filename_to_labels = json.load(f)\n\n# def filter_score_fn(self, filter_str: str, input_record: dict, output: bool) -> float | None:\n# filename = input_record[\"filename\"]\n# labels = self.filename_to_labels[filename]\n# if labels is None:\n# return None\n\n# if \"business transactions\" in filter_str:\n# return float(labels[\"mentions_transaction\"] == output)\n# elif \"first-hand discussion\" in filter_str:\n# return float(labels[\"firsthand_discussion\"] == output)\n# else:\n# return None\n\n# def map_score_fn(self, fields: list[str], input_record: dict, output: dict) -> float | None:\n# # NOTE: we score the map based on the sender and subject fields only, as summary is too subjective;\n# # we could also use an LLM judge within this function to score the summary field if desired\n# filename = input_record[\"filename\"]\n# labels = self.filename_to_labels[filename]\n# if labels is None:\n# return None\n\n# return (float(labels[\"sender\"] == output[\"sender\"]) + float(labels[\"subject\"] == output[\"subject\"])) / 2.0\n\n\nclass RealEstateListingDataset(pz.IterDataset):\n def __init__(self, listings_dir):\n super().__init__(id=\"real-estate\", schema=real_estate_listing_cols)\n self.listings_dir = listings_dir\n self.listings = sorted(os.listdir(self.listings_dir))\n self.listings = [file for file in self.listings if not file.startswith(\".\")]\n\n def __len__(self):\n return len(self.listings)\n\n def __getitem__(self, idx: int):\n # get listing\n listing = self.listings[idx]\n\n # get fields\n image_filepaths, text_content = [], None\n listing_dir = os.path.join(self.listings_dir, listing)\n for file in os.listdir(listing_dir):\n if file.endswith(\".txt\"):\n with open(os.path.join(listing_dir, file), \"rb\") as f:\n text_content = f.read().decode(\"utf-8\")\n elif file.endswith(\".png\"):\n image_filepaths.append(os.path.join(listing_dir, file))\n\n # construct and return dictionary with fields\n return {\"listing\": listing, \"text_content\": text_content, \"image_filepaths\": image_filepaths}\n\n\nif __name__ == \"__main__\":\n # parse arguments\n parser = argparse.ArgumentParser(description=\"Run a simple demo\")\n parser.add_argument(\"--viz\", default=False, action=\"store_true\", help=\"Visualize output in Gradio\")\n parser.add_argument(\"--dataset\", type=str, help=\"The path to the dataset\")\n parser.add_argument(\n \"--policy\",\n type=str,\n help=\"One of 'mincost', 'mintime', 'maxquality'\",\n default=\"maxquality\",\n )\n\n args = parser.parse_args()\n\n # The user has to indicate the dataset id and the workload\n if args.dataset is None:\n print(\"Please provide a dataset id\")\n exit(1)\n\n dataset = args.dataset\n visualize = args.viz\n policy = pz.MaxQuality()\n if args.policy == \"mincost\":\n policy = pz.MinCost()\n elif args.policy == \"mintime\":\n policy = pz.MinTime()\n elif args.policy == \"maxquality\":\n policy = pz.MaxQuality()\n else:\n print(\"Policy not supported for this demo\")\n exit(1)\n\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n # create pz plan\n plan = RealEstateListingDataset(dataset)\n plan = plan.sem_map(real_estate_text_cols, depends_on=\"text_content\")\n plan = plan.sem_map(real_estate_image_cols, depends_on=\"image_filepaths\")\n plan = plan.sem_filter(\n \"The interior is modern and attractive, and has lots of natural sunlight\",\n depends_on=[\"is_modern_and_attractive\", \"has_natural_sunlight\"],\n )\n plan = plan.filter(within_two_miles_of_mit, depends_on=\"address\")\n plan = plan.filter(in_price_range, depends_on=\"price\")\n\n # construct config and run plan\n config = pz.QueryProcessorConfig(\n policy=policy,\n available_models=[pz.Model.GPT_5_MINI],\n k=6,\n j=6,\n sample_budget=125,\n )\n data_record_collection = plan.optimize_and_run(config, validator=pz.Validator(model=pz.Model.o4_MINI))\n print(data_record_collection.to_df())\n\n # preds = data_record_collection.to_df()[\"listing\"].tolist()\n # gt_df = pd.read_csv(\"testdata/groundtruth/real-estate-eval-100.csv\")\n # labels = gt_df.listing.tolist()\n # tp, fp, fn = 0, 0, 0\n # for pred in preds:\n # if pred in labels:\n # tp += 1\n # else:\n # fp += 1\n # for label in labels:\n # if label not in preds:\n # fn += 1\n # precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0\n # recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0\n # f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0\n # print(f\"Precision: {precision:.2f}\")\n # print(f\"Recall: {recall:.4f}\")\n # print(f\"F1: {f1:.4f}\")\n\n # visualize output in Gradio\n if visualize:\n plan_str = list(data_record_collection.execution_stats.plan_strs.values())[-1]\n fst_imgs, snd_imgs, thrd_imgs, addrs, prices = [], [], [], [], []\n for record in data_record_collection:\n addrs.append(record.address)\n prices.append(record.price)\n for idx, img_name in enumerate([\"img1.png\", \"img2.png\", \"img3.png\"]):\n path = os.path.join(dataset, record.listing, img_name)\n img = Image.open(path)\n img_arr = np.asarray(img)\n if idx == 0:\n fst_imgs.append(img_arr)\n elif idx == 1:\n snd_imgs.append(img_arr)\n elif idx == 2:\n thrd_imgs.append(img_arr)\n\n with gr.Blocks() as demo:\n fst_img_blocks, snd_img_blocks, thrd_img_blocks, addr_blocks, price_blocks = [], [], [], [], []\n for fst_img, snd_img, thrd_img, addr, price in zip(fst_imgs, snd_imgs, thrd_imgs, addrs, prices):\n with gr.Row(equal_height=True):\n with gr.Column():\n fst_img_blocks.append(gr.Image(value=fst_img))\n with gr.Column():\n snd_img_blocks.append(gr.Image(value=snd_img))\n with gr.Column():\n thrd_img_blocks.append(gr.Image(value=thrd_img))\n with gr.Row():\n with gr.Column():\n addr_blocks.append(gr.Textbox(value=addr, info=\"Address\"))\n with gr.Column():\n price_blocks.append(gr.Textbox(value=price, info=\"Price\"))\n\n plan_str = list(data_record_collection.execution_stats.plan_strs.values())[0]\n gr.Textbox(value=plan_str, info=\"Query Plan\")\n\n demo.launch()\n" }, { "path": "demos/simple-demo.py", "content": "#!/usr/bin/env python3\nimport argparse\nimport os\nimport time\n\nfrom demo_core import execute_task, format_results_table\nfrom dotenv import load_dotenv\n\nimport palimpzest as pz\n\nload_dotenv()\n\ndef main():\n # parse arguments\n start_time = time.time()\n parser = argparse.ArgumentParser(description=\"Run a simple demo\")\n parser.add_argument(\"--verbose\", default=False, action=\"store_true\", help=\"Print verbose output\")\n parser.add_argument(\"--profile\", default=False, action=\"store_true\", help=\"Profile execution\")\n parser.add_argument(\"--dataset\", type=str, help=\"Path to the dataset\")\n parser.add_argument(\"--join-dataset\", type=str, help=\"Path to the join dataset (if needed)\", default=None)\n parser.add_argument(\"--task\", type=str, help=\"The task to run\")\n parser.add_argument(\n \"--execution-strategy\",\n type=str,\n help=\"The execution strategy to use. One of sequential, pipelined, parallel\",\n default=\"sequential\",\n )\n parser.add_argument(\n \"--policy\",\n type=str,\n help=\"One of 'mincost', 'mintime', 'maxquality'\",\n default=\"maxquality\",\n )\n\n args = parser.parse_args()\n\n # The user has to indicate the dataset and the task\n if args.dataset is None:\n print(\"Please provide a path to the dataset\")\n exit(1)\n if args.task is None:\n print(\"Please provide a task\")\n exit(1)\n\n # Set up execution parameters\n dataset = args.dataset\n join_dataset = args.join_dataset\n task = args.task\n verbose = args.verbose\n profile = args.profile\n\n # Set policy\n policy = pz.MaxQuality()\n if args.policy == \"mincost\":\n policy = pz.MinCost()\n elif args.policy == \"mintime\":\n policy = pz.MinTime()\n elif args.policy == \"maxquality\":\n policy = pz.MaxQuality()\n else:\n print(\"Policy not supported for this demo\")\n exit(1)\n\n if os.getenv(\"OPENAI_API_KEY\") is None and os.getenv(\"TOGETHER_API_KEY\") is None and os.getenv(\"ANTHROPIC_API_KEY\") is None:\n print(\"WARNING: OPENAI_API_KEY, TOGETHER_API_KEY, and ANTHROPIC_API_KEY are unset\")\n\n # Execute task\n records, execution_stats, cols = execute_task(\n task=task,\n dataset=dataset,\n policy=policy,\n join_dataset=join_dataset,\n verbose=verbose,\n profile=profile,\n execution_strategy=args.execution_strategy\n )\n\n # Print results\n print(f\"Policy is: {str(policy)}\")\n print(\"Executed plan:\")\n plan_str = list(execution_stats.plan_strs.values())[0]\n print(plan_str)\n end_time = time.time()\n print(\"Elapsed time:\", end_time - start_time)\n\n print(format_results_table(records, cols=cols))\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "demos/vllm-demo.py", "content": "#!/usr/bin/env python3\n\"\"\"\nMinimal demo for running a vLLM model with Palimpzest.\n\nPrerequisites:\n 1. Start a vLLM server serving a small model, e.g.:\n vllm serve Qwen/Qwen2.5-1.5B-Instruct --port 8000\n 2. Run this script:\n python demos/vllm-demo.py \\\n --api-base http://localhost:8000/v1 \\\n --model-id openai/Qwen/Qwen2.5-1.5B-Instruct\n\"\"\"\nimport argparse\nimport os\n\nfrom pydantic import BaseModel, Field\n\nimport palimpzest as pz\n\n\nclass SentimentResult(BaseModel):\n sentiment: str = Field(description=\"The sentiment of the text: positive, negative, or neutral\")\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Run a minimal vLLM demo\")\n parser.add_argument(\"--api-base\", type=str, required=True, help=\"vLLM server base URL (e.g. http://localhost:8000/v1)\")\n parser.add_argument(\"--model-id\", type=str, required=True, help=\"Model ID for litellm (e.g. openai/Qwen/Qwen2.5-1.5B-Instruct)\")\n parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Max tokens for completion\")\n parser.add_argument(\"--verbose\", action=\"store_true\", default=False)\n args = parser.parse_args()\n\n # Create the vLLM model with api_base and kwargs on the Model instance\n vllm_model = pz.Model(args.model_id, api_base=args.api_base, max_tokens=args.max_tokens)\n\n # Load the enron-tiny dataset\n data_path = os.path.join(os.path.dirname(__file__), \"..\", \"testdata\", \"enron-tiny\")\n dataset = pz.TextFileDataset(id=\"test-sentiment\", path=data_path)\n dataset = dataset.sem_map(SentimentResult, desc=\"Classify the sentiment of the text\")\n\n # Configure with vLLM model\n config = pz.QueryProcessorConfig(\n policy=pz.MaxQuality(),\n available_models=[vllm_model],\n execution_strategy=\"sequential\",\n optimizer_strategy=\"pareto\",\n verbose=args.verbose,\n )\n\n output = dataset.run(config)\n for record in output:\n print(record)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "evals/quest/eval.py", "content": "import argparse\nimport copy\nimport json\nimport os\nimport random\nimport time\n\nimport palimpzest as pz\n\n\ndef prepare_docs_for_query(items: list, gt_docs: list) -> list:\n items = copy.copy(items)\n random.shuffle(items)\n final_items = [doc for doc in items if doc[\"title\"] in gt_docs]\n while len(final_items) < 1000 and len(items) > 0:\n item = items.pop(0)\n if item not in final_items:\n final_items.append(item)\n return final_items\n\n\ndef palimpzest_run_query(query: dict, documents: list) -> list[str]:\n gt_docs = query[\"docs\"]\n items = prepare_docs_for_query(documents, gt_docs)\n\n schema = [\n {\"name\": \"title\", \"type\": str, \"desc\": \"Document title\"},\n {\"name\": \"text\", \"type\": str, \"desc\": \"Document content\"},\n ]\n\n dataset = pz.MemoryDataset(\n id=\"quest-docs\",\n vals=items,\n schema=schema,\n )\n\n query_text = query[\"query\"]\n plan = dataset.sem_filter(\n f'This document is relevant to the entity-seeking query: \"{query_text}\". '\n \"Return True if the document helps answer the query, False otherwise.\",\n depends_on=[\"text\"],\n ).project([\"title\"])\n\n config = pz.QueryProcessorConfig(\n policy=pz.MaxQuality(),\n execution_strategy=\"parallel\",\n progress=True,\n )\n output = plan.run(config)\n execution_stats = output.execution_stats\n time_secs = execution_stats.total_execution_time if execution_stats else 0.0\n cost = execution_stats.total_execution_cost if execution_stats else 0.0\n return [record[\"title\"] for record in output], time_secs, cost\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Evaluate Palimpzest on QUEST\")\n parser.add_argument(\n \"--domain\",\n type=str,\n required=True,\n choices=[\"films\", \"books\"],\n help=\"The domain to evaluate.\",\n )\n parser.add_argument(\n \"--queries\",\n type=str,\n required=True,\n help=\"Path to the file containing the queries (e.g. test.jsonl).\",\n )\n parser.add_argument(\n \"--documents\",\n type=str,\n default=\"data/documents.jsonl\",\n help=\"Path to documents.jsonl (QUEST format: title, text per line).\",\n )\n parser.add_argument(\n \"--limit\",\n type=int,\n default=None,\n help=\"Limit number of queries to evaluate (for debugging).\",\n )\n parser.add_argument(\n \"--seed\",\n type=int,\n default=42,\n help=\"Random seed for document shuffling.\",\n )\n args = parser.parse_args()\n\n random.seed(args.seed)\n\n if not os.path.exists(args.documents):\n raise FileNotFoundError(\n f\"Documents file not found: {args.documents}\\n\"\n )\n with open(args.documents) as f:\n documents = [json.loads(line) for line in f]\n\n queries = []\n with open(args.queries) as f:\n for line in f:\n d = json.loads(line)\n if d[\"metadata\"][\"domain\"] == args.domain:\n queries.append(d)\n\n if args.limit:\n queries = queries[: args.limit]\n\n results = []\n for i, query in enumerate(queries):\n print(f\"[{i + 1}/{len(queries)}] Executing query: {query['query']}\")\n pred_docs, cur_time, cur_cost = palimpzest_run_query(query, documents)\n\n gt_docs = query[\"docs\"]\n preds = set(pred_docs)\n labels = set(gt_docs)\n\n tp = sum(1 for pred in preds if pred in labels)\n fp = len(preds) - tp\n fn = sum(1 for label in labels if label not in preds)\n\n precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n\n result = {\n \"query\": query[\"query\"],\n \"predicted_docs\": pred_docs,\n \"ground_truth_docs\": gt_docs,\n \"precision\": precision,\n \"recall\": recall,\n \"f1_score\": f1,\n \"time\": cur_time,\n \"cost\": cur_cost\n }\n results.append(result)\n\n ts = int(time.time())\n out_path = f\"results_{args.domain}_{ts}.json\"\n with open(out_path, \"w\") as f:\n json.dump(results, f, indent=4)\n print(f\"\\nResults saved to {out_path}\")\n\n n = len(results)\n avg_precision = sum(r[\"precision\"] for r in results) / n\n avg_recall = sum(r[\"recall\"] for r in results) / n\n avg_f1 = sum(r[\"f1_score\"] for r in results) / n\n avg_time = sum(r[\"time\"] for r in results) / n\n avg_cost = sum(r[\"cost\"] for r in results) / n\n\n print(f\"Average Precision: {avg_precision:.4f}\")\n print(f\"Average Recall: {avg_recall:.4f}\")\n print(f\"Average F1 Score: {avg_f1:.4f}\")\n print(f\"Average Time: {avg_time:.4f}s\")\n print(f\"Average Cost: {avg_cost:.4f}$\")\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "pyproject.toml", "content": "[project]\nname = \"palimpzest\"\nversion = \"1.5.3\"\ndescription = \"Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language\"\nreadme = \"README.md\"\nrequires-python = \">=3.12\"\nkeywords = [\"relational\", \"optimization\", \"llm\", \"AI programming\", \"extraction\", \"tools\", \"document\", \"search\", \"integration\"]\nauthors = [\n {name=\"MIT DSG Semantic Management Lab\", email=\"michjc@csail.mit.edu\"},\n]\ndependencies = [\n \"anthropic>=0.79.0\",\n \"beautifulsoup4>=4.13.4\",\n \"chromadb>=1.0.15\",\n \"colorama>=0.4.6\",\n \"datasets>=4.0.0\",\n \"fastapi>=0.115.0\",\n \"google-genai>=1.0.0\",\n \"gradio>=5.26.0\",\n \"litellm>=1.81.11, <1.82.7\",\n \"numpy==2.0.2\",\n \"openai>=1.0\",\n \"pandas>=2.1.1\",\n \"pytest>=8.2.2\",\n \"pillow>=11.3.0\",\n \"prettytable>=3.9.0\",\n \"psutil==5.9.5\",\n \"PyLD>=2.0.4\",\n \"pyarrow>=20.0.0\",\n \"pypdf>=5.1.0\",\n \"pytest-mock>=3.14.0\",\n \"python-dotenv>=1.2.1\",\n \"pyyaml>=6.0.1\",\n \"requests>=2.25\",\n \"ruff>=0.9.0\",\n \"sentence-transformers==5.0.0\",\n \"setuptools>=70.1.1\",\n \"smolagents[toolkit]\",\n \"tabulate>=0.9.0\",\n \"together>=1.5.5\",\n \"tqdm~=4.66.1\",\n \"rich[jupyter]>=13.9.2\",\n]\nclassifiers=[\n \"Development Status :: 4 - Beta\", # Change as appropriate\n \"Intended Audience :: Developers\",\n \"License :: OSI Approved :: MIT License\", # Change as appropriate\n \"Programming Language :: Python :: 3\",\n \"Programming Language :: Python :: 3.8\", # Specify versions you support\n # Add more classifiers as appropriate\n]\n\n[project.optional-dependencies]\nvllm = [\n \"vllm>=0.10.1.1\",\n]\n\n[tool.setuptools]\npackage-dir = {\"\" = \"src\"}\ninclude-package-data = true\n\n[tool.setuptools.packages.find]\nwhere = [\"src\"]\nnamespaces = false\n\n[tool.setuptools.package-data]\n\"*\" = [\"*.txt\", \"*.rst\", \"*.md\", \"*.json\"]\n\n[tool.pytest.ini_options]\ntestpaths = [\"tests/pytest\"]\nfilterwarnings = [\n \"error\",\n \"ignore::DeprecationWarning\",\n \"ignore::ResourceWarning\",\n \"ignore::UserWarning\",\n]\n\n[build-system]\nrequires = [\"setuptools\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[project.urls]\nhomepage = \"https://palimpzest.org\"\nrepository = \"https://github.com/mitdbg/palimpzest/\"\ndocumentation = \"https://palimpzest.org\"\n# changelog = \"https://github.com/me/spam/blob/master/CHANGELOG.md\"\n" }, { "path": "quickstart.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"dBfyB-7Hytwy\"\n },\n \"source\": [\n \"![PZ-banner](https://palimpzest-workloads.s3.us-east-1.amazonaws.com/palimpzest-cropped.png)\\n\",\n \"\\n\",\n \"# Palimpzest Quickstart\\n\",\n \"This notebook contains a sample program to guide you through the features of the Palimpzest (PZ) library. PZ provides a high-level, declarative interface for composing and executing pipelines of semantic operators.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"2-TkUeCFx1et\"\n },\n \"source\": [\n \"## Load Private Key(s)\\n\",\n \"1. Click on the \\\"key\\\" icon on the left-hand-side of the Colab notebook.\\n\",\n \"2. In the sidebar that opens, click `+ Add new secret`\\n\",\n \" - **Note:** your secrets are not visible to anyone other than Google and your version of the notebook.\\n\",\n \"3. Enter one or more of the following keys as secrets:\\n\",\n \" - `OPENAI_API_KEY`\\n\",\n \" - `TOGETHER_API_KEY`\\n\",\n \" - You can create a `together.ai` API key [here](https://api.together.ai/) for this demo (it comes with $1 of free API requests)\\n\",\n \"4. Make sure you have toggled `Notebook access` ON\\n\",\n \"5. Execute the cell below to store these keys in notebook environment variables.\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"zmmkh1n8efxA\"\n },\n \"source\": [\n \"#### Note: for the changes to take effect, you may need to restart the session (`Runtime > Restart Session`) if you've already connected the notebook to a runtime\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"-DgUrHNtZu0z\"\n },\n \"outputs\": [],\n \"source\": [\n \"from google.colab import userdata\\n\",\n \"import os\\n\",\n \"\\n\",\n \"# set environment variables\\n\",\n \"def set_api_key_from_secret(key_name):\\n\",\n \" try:\\n\",\n \" os.environ[key_name] = userdata.get(key_name)\\n\",\n \" except:\\n\",\n \" pass\\n\",\n \"\\n\",\n \"set_api_key_from_secret('OPENAI_API_KEY')\\n\",\n \"set_api_key_from_secret('TOGETHER_API_KEY')\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"HNFA4gTzxvE2\"\n },\n \"source\": [\n \"## Install Palimpzest\\n\",\n \"First, let's install the Palimpzest package. This may take a few minutes. **PIP dependency error messages are expected and can be ignored.**\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"collapsed\": true,\n \"id\": \"4AxQGqXIyXsP\"\n },\n \"outputs\": [],\n \"source\": [\n \"!pip install palimpzest==0.7.6\\n\",\n \"!pip install --upgrade pyarrow\\n\",\n \"!pip install chromadb==0.6.3\\n\",\n \"import palimpzest as pz\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"qSAC96Rb-Ggy\"\n },\n \"source\": [\n \"## Download Test Files\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"cUwsu8XOzgJd\"\n },\n \"source\": [\n \"Next, we'll download the dataset we need for this demo:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"collapsed\": true,\n \"id\": \"IXv-pxMhx0i1\"\n },\n \"outputs\": [],\n \"source\": [\n \"# download tar files with testdata\\n\",\n \"!wget -nc https://people.csail.mit.edu/gerarvit/PalimpzestData/enron-tiny.tar.gz\\n\",\n \"!wget -nc wget -nc https://people.csail.mit.edu/gerarvit/PalimpzestData/real-estate-eval-5.tar.gz\\n\",\n \"!wget -nc https://palimpzest-workloads.s3.us-east-1.amazonaws.com/chroma-biodex.tar.gz\\n\",\n \"\\n\",\n \"# open tar files\\n\",\n \"!tar -xzf enron-tiny.tar.gz\\n\",\n \"!tar -xzf real-estate-eval-5.tar.gz\\n\",\n \"!tar -xzf chroma-biodex.tar.gz\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"fw5mmyAY_EaS\"\n },\n \"source\": [\n \"# First PZ Program: Filtering Enron Emails\\n\",\n \"For this demo, we will work with a small subset of the Enron Email Dataset to identify emails matching some search criteria.\\n\",\n \"\\n\",\n \"We are going to use Palimpzest to perform the following tasks:\\n\",\n \"1. Load the text files that contain the emails. (Each `.txt` file contains a single email).\\n\",\n \"2. Compute the sender, subject, and date of each email.\\n\",\n \"3. Filter the emails for ones that mention a vacation plan and were sent in the month of July.\\n\",\n \"\\n\",\n \"We can compose these tasks into a PZ program as follows:\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"collapsed\": true,\n \"id\": \"lE8xx1s7xoQQ\"\n },\n \"outputs\": [],\n \"source\": [\n \"# define the fields we wish to compute\\n\",\n \"email_cols = [\\n\",\n \" {\\\"name\\\": \\\"sender\\\", \\\"type\\\": str, \\\"desc\\\": \\\"The email address of the sender\\\"},\\n\",\n \" {\\\"name\\\": \\\"subject\\\", \\\"type\\\": str, \\\"desc\\\": \\\"The subject of the email\\\"},\\n\",\n \" {\\\"name\\\": \\\"date\\\", \\\"type\\\": str, \\\"desc\\\": \\\"The date the email was sent\\\"},\\n\",\n \"]\\n\",\n \"\\n\",\n \"# lazily construct the computation to get emails about holidays sent in July\\n\",\n \"dataset = pz.Dataset(\\\"enron-tiny/\\\")\\n\",\n \"dataset = dataset.sem_add_columns(email_cols)\\n\",\n \"dataset = dataset.sem_filter(\\\"The email was sent in July\\\")\\n\",\n \"dataset = dataset.sem_filter(\\\"The email is about holidays\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"ZRYDgD3RsMCo\"\n },\n \"source\": [\n \"First, we define the set of columns we want to compute in `email_cols`.\\n\",\n \"\\n\",\n \"Next, we create a dataset by simply constructing `pz.Dataset()` with to the path to our files.\\n\",\n \"\\n\",\n \"We then instruct PZ to compute the email columns with a call to `sem_add_columns()`.\\n\",\n \"\\n\",\n \"Finally, we apply our two natural language filters with `sem_filter()`.\\n\",\n \"\\n\",\n \"**Note:** due to PZ's lazy execution, the code above will not execute the PZ program. It simply defines the semantic computation graph.\\n\",\n \"\\n\",\n \"In the next cell, we execute the PZ program with the goal of optimizing for quality:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"cSbS7uC7tUyS\"\n },\n \"outputs\": [],\n \"source\": [\n \"# execute the computation w/the MaxQuality policy\\n\",\n \"config = pz.QueryProcessorConfig(policy=pz.MaxQuality(), execution_strategy=\\\"parallel\\\", progress=True)\\n\",\n \"output = dataset.run(config)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"9hAjI6JrtbIU\"\n },\n \"source\": [\n \"Once our pipeline completes, we can convert the output to a Pandas DataFrame:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"cyTrabGGtaZq\"\n },\n \"outputs\": [],\n \"source\": [\n \"# display output (if using Jupyter, otherwise use print(output_df))\\n\",\n \"output_df = output.to_df(cols=[\\\"date\\\", \\\"sender\\\", \\\"subject\\\"])\\n\",\n \"display(output_df)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"55DHU5XNAYBR\"\n },\n \"source\": [\n \"Furthermore, Palimpzest provides a detailed report of the execution, with statistics about the runtime and cost of each operation, as well as the final plan that PZ executed.\\n\",\n \"\\n\",\n \"These statistics are stored in `output.execution_stats`:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"ottmnW4OAhXv\"\n },\n \"outputs\": [],\n \"source\": [\n \"print(f\\\"Optimization Time: {output.execution_stats.optimization_time:.2f}s\\\")\\n\",\n \"print(f\\\"Optimization Cost: ${output.execution_stats.optimization_cost:.3f}\\\")\\n\",\n \"print(\\\"---\\\")\\n\",\n \"print(f\\\"Plan Execution Time: {output.execution_stats.plan_execution_time:.2f}s\\\")\\n\",\n \"print(f\\\"Plan Execution Cost: ${output.execution_stats.plan_execution_cost:.3f}\\\")\\n\",\n \"\\n\",\n \"print(\\\"Final plan executed:\\\")\\n\",\n \"print(\\\"---\\\")\\n\",\n \"final_plan_id = list(output.execution_stats.plan_strs.keys())[-1]\\n\",\n \"print(output.execution_stats.plan_strs[final_plan_id])\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"ojm-qRxMyO0i\"\n },\n \"source\": [\n \"# Second PZ Program: Multi-Modal Data Processing\\n\",\n \"\\n\",\n \"For our next demo, we will work with a small dataset of five real estate listings to search for properties of interest.\\n\",\n \"\\n\",\n \"We are going to use Palimpzest to execute the following pipeline.\\n\",\n \"1. Load the images and text description for each listing\\n\",\n \"2. Compute the price and address of each listing from the text description\\n\",\n \"3. Filter for homes within our price range\\n\",\n \"4. Filter for homes that look modern and attractive\\n\",\n \"\\n\",\n \"Let's take a moment to visualize the homes in our dataset:\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"zUHkZDwC6EdC\"\n },\n \"outputs\": [],\n \"source\": [\n \"from PIL import Image\\n\",\n \"import numpy as np\\n\",\n \"import gradio as gr\\n\",\n \"\\n\",\n \"# Boilerplate code to build our visualization\\n\",\n \"fst_imgs, snd_imgs, thrd_imgs, texts = [], [], [], []\\n\",\n \"for idx in range(1, 6):\\n\",\n \" listing = f\\\"listing{idx}\\\"\\n\",\n \" with open(os.path.join(\\\"real-estate-eval-5\\\", listing, \\\"listing-text.txt\\\")) as f:\\n\",\n \" texts.append(f.read())\\n\",\n \" for idx, img_name in enumerate([\\\"img1.png\\\", \\\"img2.png\\\", \\\"img3.png\\\"]):\\n\",\n \" path = os.path.join(\\\"real-estate-eval-5\\\", listing, img_name)\\n\",\n \" img = Image.open(path)\\n\",\n \" img_arr = np.asarray(img)\\n\",\n \" if idx == 0:\\n\",\n \" fst_imgs.append(img_arr)\\n\",\n \" elif idx == 1:\\n\",\n \" snd_imgs.append(img_arr)\\n\",\n \" elif idx == 2:\\n\",\n \" thrd_imgs.append(img_arr)\\n\",\n \"\\n\",\n \"with gr.Blocks() as demo:\\n\",\n \" fst_img_blocks, snd_img_blocks, thrd_img_blocks, text_blocks = [], [], [], []\\n\",\n \" for fst_img, snd_img, thrd_img, text in zip(fst_imgs, snd_imgs, thrd_imgs, texts):\\n\",\n \" with gr.Row(equal_height=True):\\n\",\n \" with gr.Column():\\n\",\n \" fst_img_blocks.append(gr.Image(value=fst_img))\\n\",\n \" with gr.Column():\\n\",\n \" snd_img_blocks.append(gr.Image(value=snd_img))\\n\",\n \" with gr.Column():\\n\",\n \" thrd_img_blocks.append(gr.Image(value=thrd_img))\\n\",\n \" with gr.Row():\\n\",\n \" with gr.Column():\\n\",\n \" text_blocks.append(gr.Textbox(value=text, info=\\\"Text Description\\\"))\\n\",\n \"\\n\",\n \"demo.launch()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"Yg4yRYx26ecr\"\n },\n \"outputs\": [],\n \"source\": [\n \"demo.close()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"GpNe3bFk6FMD\"\n },\n \"source\": [\n \"As a first step, we need to write a custom `pz.DataReader` to enable PZ to load our data properly:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"2FcqpZySyWbl\"\n },\n \"outputs\": [],\n \"source\": [\n \"from palimpzest.core.lib.fields import ImageFilepathField, ListField\\n\",\n \"\\n\",\n \"# we first define the schema for each record output by the DataReader\\n\",\n \"real_estate_listing_cols = [\\n\",\n \" {\\\"name\\\": \\\"listing\\\", \\\"type\\\": str, \\\"desc\\\": \\\"The name of the listing\\\"},\\n\",\n \" {\\\"name\\\": \\\"text_content\\\", \\\"type\\\": str, \\\"desc\\\": \\\"The content of the listing's text description\\\"},\\n\",\n \" {\\\"name\\\": \\\"image_filepaths\\\", \\\"type\\\": ListField(ImageFilepathField), \\\"desc\\\": \\\"A list of the filepaths for each image of the listing\\\"},\\n\",\n \"]\\n\",\n \"\\n\",\n \"# we then implement the DataReader\\n\",\n \"class RealEstateListingReader(pz.DataReader):\\n\",\n \" def __init__(self, listings_dir):\\n\",\n \" super().__init__(schema=real_estate_listing_cols)\\n\",\n \" self.listings_dir = listings_dir\\n\",\n \" self.listings = sorted(os.listdir(self.listings_dir))\\n\",\n \"\\n\",\n \" def __len__(self):\\n\",\n \" return len(self.listings)\\n\",\n \"\\n\",\n \" def __getitem__(self, idx: int):\\n\",\n \" # get listing\\n\",\n \" listing = self.listings[idx]\\n\",\n \"\\n\",\n \" # get fields\\n\",\n \" image_filepaths, text_content = [], None\\n\",\n \" listing_dir = os.path.join(self.listings_dir, listing)\\n\",\n \" for file in os.listdir(listing_dir):\\n\",\n \" if file.endswith(\\\".txt\\\"):\\n\",\n \" with open(os.path.join(listing_dir, file), \\\"rb\\\") as f:\\n\",\n \" text_content = f.read().decode(\\\"utf-8\\\")\\n\",\n \" elif file.endswith(\\\".png\\\"):\\n\",\n \" image_filepaths.append(os.path.join(listing_dir, file))\\n\",\n \"\\n\",\n \" # construct and return dictionary with fields\\n\",\n \" return {\\\"listing\\\": listing, \\\"text_content\\\": text_content, \\\"image_filepaths\\\": image_filepaths}\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"Q45qFYcM0N9w\"\n },\n \"source\": [\n \"Every `pz.DataReader` must have the following:\\n\",\n \"1. A `schema` defining the fields present in each output record\\n\",\n \"2. A `__len__()` function which returns the number of items in the dataset\\n\",\n \"3. A `__getitem__(idx)` function which returns the `idx`th item in the dataset\\n\",\n \"\\n\",\n \"Once we've implemented the `pz.DataReader`, we can compose our PZ program as follows:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"1qOI9WOY0X4_\"\n },\n \"outputs\": [],\n \"source\": [\n \"# schema for computing the address and price of each home\\n\",\n \"real_estate_text_cols = [\\n\",\n \" {\\\"name\\\": \\\"address\\\", \\\"type\\\": str, \\\"desc\\\": \\\"The address of the property\\\"},\\n\",\n \" {\\\"name\\\": \\\"price\\\", \\\"type\\\": int | float, \\\"desc\\\": \\\"The listed price of the property\\\"},\\n\",\n \"]\\n\",\n \"\\n\",\n \"# define a UDF for filtering based on a price range\\n\",\n \"def in_price_range(record: dict):\\n\",\n \" try:\\n\",\n \" price = record[\\\"price\\\"]\\n\",\n \" if isinstance(price, str):\\n\",\n \" price = price.strip()\\n\",\n \" price = int(price.replace(\\\"$\\\", \\\"\\\").replace(\\\",\\\", \\\"\\\"))\\n\",\n \" return 6e5 < price <= 2e6\\n\",\n \" except Exception:\\n\",\n \" return False\\n\",\n \"\\n\",\n \"# construct our PZ program to filter for listings matching our search criteria\\n\",\n \"ds = pz.Dataset(RealEstateListingReader(\\\"real-estate-eval-5\\\"))\\n\",\n \"ds = ds.sem_add_columns(real_estate_text_cols, depends_on=\\\"text_content\\\")\\n\",\n \"ds = ds.sem_filter(\\n\",\n \" \\\"The interior is modern and attractive, and has lots of natural sunlight\\\",\\n\",\n \" depends_on=\\\"image_filepaths\\\",\\n\",\n \")\\n\",\n \"ds = ds.filter(in_price_range, depends_on=\\\"price\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"eitGXQCS0YRY\"\n },\n \"source\": [\n \"First, we write a schema for the `address` and `price` fields we wish to compute.\\n\",\n \"\\n\",\n \"Next, we write a UDF to filter for homes based on our price range.\\n\",\n \"\\n\",\n \"Then we compose our program by:\\n\",\n \"1. Constructing our `pz.DataReader` with the real estate data\\n\",\n \"2. Using `sem_add_columns()` to compute the `address` and `price`\\n\",\n \"3. Using a `sem_filter()` to filter for modern homes with lots of sunlight\\n\",\n \"4. Using our UDF to filter for homes based on our price range\\n\",\n \"\\n\",\n \"We now execute the program:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"uFmakjcQ4W5o\"\n },\n \"outputs\": [],\n \"source\": [\n \"# execute the computation w/the MaxQuality policy\\n\",\n \"config = pz.QueryProcessorConfig(policy=pz.MaxQuality(), execution_strategy=\\\"parallel\\\", progress=True)\\n\",\n \"output = ds.run(config)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"YHesFcjc4snL\"\n },\n \"source\": [\n \"Now let's take a look at our output:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"JAxeR_R54vuY\"\n },\n \"outputs\": [],\n \"source\": [\n \"from PIL import Image\\n\",\n \"import numpy as np\\n\",\n \"import gradio as gr\\n\",\n \"\\n\",\n \"demo.close()\\n\",\n \"\\n\",\n \"# Boilerplate code to build our visualization\\n\",\n \"fst_imgs, snd_imgs, thrd_imgs, addrs, prices = [], [], [], [], []\\n\",\n \"for record in output:\\n\",\n \" addrs.append(record.address)\\n\",\n \" prices.append(record.price)\\n\",\n \" for idx, img_name in enumerate([\\\"img1.png\\\", \\\"img2.png\\\", \\\"img3.png\\\"]):\\n\",\n \" path = os.path.join(\\\"real-estate-eval-5\\\", record.listing, img_name)\\n\",\n \" img = Image.open(path)\\n\",\n \" img_arr = np.asarray(img)\\n\",\n \" if idx == 0:\\n\",\n \" fst_imgs.append(img_arr)\\n\",\n \" elif idx == 1:\\n\",\n \" snd_imgs.append(img_arr)\\n\",\n \" elif idx == 2:\\n\",\n \" thrd_imgs.append(img_arr)\\n\",\n \"\\n\",\n \"with gr.Blocks() as demo:\\n\",\n \" fst_img_blocks, snd_img_blocks, thrd_img_blocks, addr_blocks, price_blocks = [], [], [], [], []\\n\",\n \" for fst_img, snd_img, thrd_img, addr, price in zip(fst_imgs, snd_imgs, thrd_imgs, addrs, prices):\\n\",\n \" with gr.Row(equal_height=True):\\n\",\n \" with gr.Column():\\n\",\n \" fst_img_blocks.append(gr.Image(value=fst_img))\\n\",\n \" with gr.Column():\\n\",\n \" snd_img_blocks.append(gr.Image(value=snd_img))\\n\",\n \" with gr.Column():\\n\",\n \" thrd_img_blocks.append(gr.Image(value=thrd_img))\\n\",\n \" with gr.Row():\\n\",\n \" with gr.Column():\\n\",\n \" addr_blocks.append(gr.Textbox(value=addr, info=\\\"Address\\\"))\\n\",\n \" with gr.Column():\\n\",\n \" price_blocks.append(gr.Textbox(value=price, info=\\\"Price\\\"))\\n\",\n \"\\n\",\n \" plan_str = list(output.execution_stats.plan_strs.values())[0]\\n\",\n \" gr.Textbox(value=plan_str, info=\\\"Query Plan\\\")\\n\",\n \"\\n\",\n \"demo.launch()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"XpCo8uF54sPi\"\n },\n \"outputs\": [],\n \"source\": [\n \"demo.close()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"VAgG4wBKyZM0\"\n },\n \"source\": [\n \"# Third PZ Program: Optimizing a Biomedical Classification Pipeline\\n\",\n \"\\n\",\n \"For our final demo, we will work with a subset of the BioDEX dataset.\\n\",\n \"\\n\",\n \"Each input in the dataset is a medical report describing an adverse reaction a patient had in response to taking one or more drugs.\\n\",\n \"\\n\",\n \"The goal is to correctly predict the reactions experienced by the patient by matching them to a database of ~24,300 official medical reaction terms.\\n\",\n \"\\n\",\n \"We are going to use Palimpzest to implement the following pipeline:\\n\",\n \"1. Load a medical report\\n\",\n \"2. Compute a list of reactions mentioned in the report\\n\",\n \"3. Retrieve the most similar reaction terms from a vector database with embeddings for each of the ~24,300 official terms\\n\",\n \"4. Re-rank the list of official terms based on their relevance\\n\",\n \"\\n\",\n \"First, we will once again create a `pz.DataReader` to load the medical reports:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"_CeZ9Ib1yY1n\"\n },\n \"outputs\": [],\n \"source\": [\n \"import datasets\\n\",\n \"from functools import partial\\n\",\n \"\\n\",\n \"# define the schema for records returned by the DataReader\\n\",\n \"biodex_entry_cols = [\\n\",\n \" {\\\"name\\\": \\\"pmid\\\", \\\"type\\\": str, \\\"desc\\\": \\\"The PubMed ID of the medical paper\\\"},\\n\",\n \" {\\\"name\\\": \\\"title\\\", \\\"type\\\": str, \\\"desc\\\": \\\"The title of the medical paper\\\"},\\n\",\n \" {\\\"name\\\": \\\"abstract\\\", \\\"type\\\": str, \\\"desc\\\": \\\"The abstract of the medical paper\\\"},\\n\",\n \" {\\\"name\\\": \\\"fulltext\\\", \\\"type\\\": str, \\\"desc\\\": \\\"The full text of the medical paper, which contains information relevant for creating a drug safety report.\\\"},\\n\",\n \"]\\n\",\n \"\\n\",\n \"# implement the DataReader\\n\",\n \"class BiodexReader(pz.DataReader):\\n\",\n \" def __init__(\\n\",\n \" self,\\n\",\n \" rp_at_k: int = 5,\\n\",\n \" num_samples: int = 10,\\n\",\n \" split: str = \\\"test\\\",\\n\",\n \" shuffle: bool = True,\\n\",\n \" seed: int = 42,\\n\",\n \" ):\\n\",\n \" super().__init__(biodex_entry_cols)\\n\",\n \"\\n\",\n \" self.dataset = datasets.load_dataset(\\\"BioDEX/BioDEX-Reactions\\\", split=split).to_pandas()\\n\",\n \" if shuffle:\\n\",\n \" self.dataset = self.dataset.sample(n=num_samples, random_state=seed).to_dict(orient=\\\"records\\\")\\n\",\n \" else:\\n\",\n \" self.dataset = self.dataset.to_dict(orient=\\\"records\\\")[:num_samples]\\n\",\n \"\\n\",\n \" self.rp_at_k = rp_at_k\\n\",\n \" self.num_samples = num_samples\\n\",\n \" self.shuffle = shuffle\\n\",\n \" self.seed = seed\\n\",\n \" self.split = split\\n\",\n \"\\n\",\n \" def compute_label(self, entry: dict) -> dict:\\n\",\n \" \\\"\\\"\\\"Compute the label for a BioDEX report given its entry in the dataset.\\\"\\\"\\\"\\n\",\n \" reactions_lst = [\\n\",\n \" reaction.strip().lower().replace(\\\"'\\\", \\\"\\\").replace(\\\"^\\\", \\\"\\\")\\n\",\n \" for reaction in entry[\\\"reactions\\\"].split(\\\",\\\")\\n\",\n \" ]\\n\",\n \" label_dict = {\\n\",\n \" \\\"reactions\\\": reactions_lst,\\n\",\n \" \\\"reaction_labels\\\": reactions_lst,\\n\",\n \" \\\"ranked_reaction_labels\\\": reactions_lst,\\n\",\n \" }\\n\",\n \" return label_dict\\n\",\n \"\\n\",\n \" @staticmethod\\n\",\n \" def rank_precision_at_k(preds, targets, k: int):\\n\",\n \" if preds is None:\\n\",\n \" return 0.0\\n\",\n \"\\n\",\n \" try:\\n\",\n \" # lower-case each list\\n\",\n \" preds = [pred.strip().lower().replace(\\\"'\\\", \\\"\\\").replace(\\\"^\\\", \\\"\\\") for pred in preds]\\n\",\n \" targets = set([target.strip().lower().replace(\\\"'\\\", \\\"\\\").replace(\\\"^\\\", \\\"\\\") for target in targets])\\n\",\n \"\\n\",\n \" # compute rank-precision at k\\n\",\n \" rn = len(targets)\\n\",\n \" denom = min(k, rn)\\n\",\n \" total = 0.0\\n\",\n \" for i in range(k):\\n\",\n \" total += preds[i] in targets if i < len(preds) else 0.0\\n\",\n \"\\n\",\n \" return total / denom\\n\",\n \"\\n\",\n \" except Exception:\\n\",\n \" return 0.0\\n\",\n \"\\n\",\n \" @staticmethod\\n\",\n \" def term_recall(preds, targets):\\n\",\n \" if preds is None:\\n\",\n \" return 0.0\\n\",\n \"\\n\",\n \" try:\\n\",\n \" # normalize terms in each list\\n\",\n \" pred_terms = set([\\n\",\n \" term.strip()\\n\",\n \" for pred in preds\\n\",\n \" for term in pred.lower().replace(\\\"'\\\", \\\"\\\").replace(\\\"^\\\", \\\"\\\").split(\\\" \\\")\\n\",\n \" ])\\n\",\n \" target_terms = ([\\n\",\n \" term.strip()\\n\",\n \" for target in targets\\n\",\n \" for term in target.lower().replace(\\\"'\\\", \\\"\\\").replace(\\\"^\\\", \\\"\\\").split(\\\" \\\")\\n\",\n \" ])\\n\",\n \"\\n\",\n \" # compute term recall and return\\n\",\n \" intersect = pred_terms.intersection(target_terms)\\n\",\n \" term_recall = len(intersect) / len(target_terms)\\n\",\n \"\\n\",\n \" return term_recall\\n\",\n \"\\n\",\n \" except Exception:\\n\",\n \" return 0.0\\n\",\n \"\\n\",\n \" def __len__(self):\\n\",\n \" return len(self.dataset)\\n\",\n \"\\n\",\n \" def __getitem__(self, idx: int):\\n\",\n \" # get entry\\n\",\n \" entry = self.dataset[idx]\\n\",\n \"\\n\",\n \" # get input fields\\n\",\n \" pmid = entry[\\\"pmid\\\"]\\n\",\n \" title = entry[\\\"title\\\"]\\n\",\n \" abstract = entry[\\\"abstract\\\"]\\n\",\n \" fulltext = entry[\\\"fulltext\\\"]\\n\",\n \"\\n\",\n \" # create item with fields\\n\",\n \" item = {\\\"fields\\\": {}, \\\"labels\\\": {}, \\\"score_fn\\\": {}}\\n\",\n \" item[\\\"fields\\\"][\\\"pmid\\\"] = pmid\\n\",\n \" item[\\\"fields\\\"][\\\"title\\\"] = title\\n\",\n \" item[\\\"fields\\\"][\\\"abstract\\\"] = abstract\\n\",\n \" item[\\\"fields\\\"][\\\"fulltext\\\"] = fulltext\\n\",\n \"\\n\",\n \" if self.split == \\\"train\\\":\\n\",\n \" # add label info\\n\",\n \" item[\\\"labels\\\"] = self.compute_label(entry)\\n\",\n \"\\n\",\n \" # add scoring functions for list fields\\n\",\n \" rank_precision_at_k = partial(BiodexReader.rank_precision_at_k, k=self.rp_at_k)\\n\",\n \" item[\\\"score_fn\\\"][\\\"reactions\\\"] = BiodexReader.term_recall\\n\",\n \" item[\\\"score_fn\\\"][\\\"reaction_labels\\\"] = BiodexReader.term_recall\\n\",\n \" item[\\\"score_fn\\\"][\\\"ranked_reaction_labels\\\"] = rank_precision_at_k\\n\",\n \"\\n\",\n \" return item\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"fyr71cs3EQpl\"\n },\n \"source\": [\n \"There are a few new features of this `pz.DataReader` which are needed for the optimization process:\\n\",\n \"1. `__getitem__()` returns a dictionary with top-level keys `{\\\"fields\\\", \\\"labels\\\", \\\"score_fn\\\"}`\\n\",\n \"2. `fields` contains the data emitted by the `pz.DataReader`\\n\",\n \"3. (for `train` data only): `labels` contains the expected results for each output field\\n\",\n \"4. (for `train` data only): `score_fn` contains scoring functions for each output field\\n\",\n \"\\n\",\n \"Once we've defined our `pz.DataReader`, we can create our training and test datasets:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"aHYqdDrlG8zP\"\n },\n \"outputs\": [],\n \"source\": [\n \"SEED = 123\\n\",\n \"\\n\",\n \"# create train and test datasets; and validator\\n\",\n \"train_datareader = BiodexReader(split=\\\"train\\\", seed=SEED)\\n\",\n \"test_datareader = BiodexReader(split=\\\"test\\\", num_samples=20, seed=SEED)\\n\",\n \"validator = pz.Validator(train_datareader, None)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"lHaVbQHiG-Rf\"\n },\n \"source\": [\n \"We now implement the logic for the `sem_topk` operator for you. It fetches the five most similar medical terms for each reaction computed by PZ, sorts them based on similarity, and then returns the final top-k most similar terms.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"JE3scuaXI6HB\"\n },\n \"outputs\": [],\n \"source\": [\n \"import chromadb\\n\",\n \"from chromadb.utils.embedding_functions.openai_embedding_function import OpenAIEmbeddingFunction\\n\",\n \"\\n\",\n \"# load index [text-embedding-3-small]\\n\",\n \"chroma_client = chromadb.PersistentClient(\\\".chroma-biodex\\\")\\n\",\n \"openai_ef = OpenAIEmbeddingFunction(\\n\",\n \" api_key=os.environ[\\\"OPENAI_API_KEY\\\"],\\n\",\n \" model_name=\\\"text-embedding-3-small\\\",\\n\",\n \")\\n\",\n \"index = chroma_client.get_collection(\\\"biodex-reaction-terms\\\", embedding_function=openai_ef)\\n\",\n \"\\n\",\n \"def search_func(index: chromadb.Collection, query: list[list[float]], k: int) -> list[str]:\\n\",\n \" # execute query with embeddings\\n\",\n \" results = index.query(query, n_results=5)\\n\",\n \"\\n\",\n \" # get list of result terms with their cosine similarity scores\\n\",\n \" final_results = []\\n\",\n \" for query_docs, query_distances in zip(results[\\\"documents\\\"], results[\\\"distances\\\"]):\\n\",\n \" for doc, dist in zip(query_docs, query_distances):\\n\",\n \" cosine_similarity = 1 - dist\\n\",\n \" final_results.append({\\\"content\\\": doc, \\\"similarity\\\": cosine_similarity})\\n\",\n \"\\n\",\n \" # sort the results by similarity score\\n\",\n \" sorted_results = sorted(final_results, key=lambda result: result[\\\"similarity\\\"], reverse=True)\\n\",\n \"\\n\",\n \" # remove duplicates\\n\",\n \" sorted_results_set = set()\\n\",\n \" final_sorted_results = []\\n\",\n \" for result in sorted_results:\\n\",\n \" if result[\\\"content\\\"] not in sorted_results_set:\\n\",\n \" sorted_results_set.add(result[\\\"content\\\"])\\n\",\n \" final_sorted_results.append(result[\\\"content\\\"])\\n\",\n \"\\n\",\n \" # return the top-k similar results and generation stats\\n\",\n \" return {\\\"reaction_labels\\\": final_sorted_results[:k]}\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"dCYi92YCJSuq\"\n },\n \"source\": [\n \"Finally, we can construct our PZ program:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"Z3UkmjQFBSjc\"\n },\n \"outputs\": [],\n \"source\": [\n \"# define the schema for each computation in our program\\n\",\n \"biodex_reactions_cols = [\\n\",\n \" {\\\"name\\\": \\\"reactions\\\", \\\"type\\\": list[str], \\\"desc\\\": \\\"The list of all medical conditions experienced by the patient as discussed in the report. Try to provide as many relevant medical conditions as possible.\\\"},\\n\",\n \"]\\n\",\n \"biodex_reaction_labels_cols = [\\n\",\n \" {\\\"name\\\": \\\"reaction_labels\\\", \\\"type\\\": list[str], \\\"desc\\\": \\\"Official terms for medical conditions listed in `reactions`\\\"},\\n\",\n \"]\\n\",\n \"biodex_ranked_reactions_labels_cols = [\\n\",\n \" {\\\"name\\\": \\\"ranked_reaction_labels\\\", \\\"type\\\": list[str], \\\"desc\\\": \\\"The ranked list of medical conditions experienced by the patient. The most relevant label occurs first in the list. Be sure to rank ALL of the inputs.\\\"},\\n\",\n \"]\\n\",\n \"\\n\",\n \"\\n\",\n \"# construct pz plan\\n\",\n \"plan = pz.Dataset(test_datareader)\\n\",\n \"plan = plan.sem_add_columns(biodex_reactions_cols)\\n\",\n \"plan = plan.sem_topk(\\n\",\n \" index=index,\\n\",\n \" search_func=search_func,\\n\",\n \" search_attr=\\\"reactions\\\",\\n\",\n \" output_attrs=biodex_reaction_labels_cols,\\n\",\n \")\\n\",\n \"plan = plan.sem_add_columns(biodex_ranked_reactions_labels_cols, depends_on=[\\\"title\\\", \\\"abstract\\\", \\\"fulltext\\\", \\\"reaction_labels\\\"])\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"yvFZO-LKKq3F\"\n },\n \"source\": [\n \"First, let's execute our plan without training data and score our performance:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"khhJJUXTLNDz\"\n },\n \"outputs\": [],\n \"source\": [\n \"def score_output(output, seed):\\n\",\n \" # score output\\n\",\n \" test_dataset = datasets.load_dataset(\\\"BioDEX/BioDEX-Reactions\\\", split=\\\"test\\\").to_pandas()\\n\",\n \" test_dataset = test_dataset.sample(n=20, random_state=seed).to_dict(orient=\\\"records\\\")\\n\",\n \"\\n\",\n \" # construct mapping from pmid --> label (field, value) pairs\\n\",\n \" def compute_target_record(entry):\\n\",\n \" reactions_lst = [\\n\",\n \" reaction.strip().lower().replace(\\\"'\\\", \\\"\\\").replace(\\\"^\\\", \\\"\\\")\\n\",\n \" for reaction in entry[\\\"reactions\\\"].split(\\\",\\\")\\n\",\n \" ]\\n\",\n \" label_dict = {\\\"ranked_reaction_labels\\\": reactions_lst}\\n\",\n \" return label_dict\\n\",\n \"\\n\",\n \" label_fields_to_values = {\\n\",\n \" entry[\\\"pmid\\\"]: compute_target_record(entry) for entry in test_dataset\\n\",\n \" }\\n\",\n \"\\n\",\n \" def rank_precision_at_k(preds: list, targets: list, k: int):\\n\",\n \" if preds is None:\\n\",\n \" return 0.0\\n\",\n \"\\n\",\n \" # lower-case each list\\n\",\n \" preds = [pred.lower().replace(\\\"'\\\", \\\"\\\").replace(\\\"^\\\", \\\"\\\") for pred in preds]\\n\",\n \" targets = set([target.lower().replace(\\\"'\\\", \\\"\\\").replace(\\\"^\\\", \\\"\\\") for target in targets])\\n\",\n \"\\n\",\n \" # compute rank-precision at k\\n\",\n \" rn = len(targets)\\n\",\n \" denom = min(k, rn)\\n\",\n \" total = 0.0\\n\",\n \" for i in range(k):\\n\",\n \" total += preds[i] in targets if i < len(preds) else 0.0\\n\",\n \"\\n\",\n \" return total / denom\\n\",\n \"\\n\",\n \" def compute_avg_rp_at_k(records, k=5):\\n\",\n \" total_rp_at_k = 0\\n\",\n \" bad = 0\\n\",\n \" for record in records:\\n\",\n \" pmid = record['pmid']\\n\",\n \" preds = record['ranked_reaction_labels']\\n\",\n \" targets = label_fields_to_values[pmid]['ranked_reaction_labels']\\n\",\n \" try:\\n\",\n \" total_rp_at_k += rank_precision_at_k(preds, targets, k)\\n\",\n \" except Exception:\\n\",\n \" bad += 1\\n\",\n \"\\n\",\n \" return total_rp_at_k / len(records), bad\\n\",\n \"\\n\",\n \" rp_at_k, bad = compute_avg_rp_at_k([record.to_dict() for record in output], k=5)\\n\",\n \" final_plan_id = list(output.execution_stats.plan_stats.keys())[0]\\n\",\n \" final_plan_str = output.execution_stats.plan_strs[final_plan_id]\\n\",\n \" print(\\\"---\\\")\\n\",\n \" print(\\\"#########################\\\")\\n\",\n \" print(f\\\"##### RP@5: {rp_at_k:.5f} #####\\\")\\n\",\n \" print(\\\"#########################\\\")\\n\",\n \" print(\\\"---\\\")\\n\",\n \" print(f\\\"Optimization time: {output.execution_stats.optimization_time:.2f}s\\\")\\n\",\n \" print(f\\\"Optimization cost: ${output.execution_stats.optimization_cost:.3f}\\\")\\n\",\n \" print(\\\"---\\\")\\n\",\n \" print(f\\\"Plan exec. time: {output.execution_stats.plan_execution_time:.2f}s\\\")\\n\",\n \" print(f\\\"Plan exec. cost: ${output.execution_stats.plan_execution_cost:.3f}\\\")\\n\",\n \" print(\\\"---\\\")\\n\",\n \" print(f\\\"Total time: {output.execution_stats.total_execution_time:.2f}s\\\")\\n\",\n \" print(f\\\"Total Cost: ${output.execution_stats.total_execution_cost:.3f}\\\")\\n\",\n \" print(\\\"---\\\")\\n\",\n \" print(\\\"Final Plan:\\\")\\n\",\n \" print(final_plan_str)\\n\",\n \"\\n\",\n \"import logging\\n\",\n \"logger = logging.getLogger()\\n\",\n \"logger.disabled = True\\n\",\n \"\\n\",\n \"# execute pz plan\\n\",\n \"config = pz.QueryProcessorConfig(\\n\",\n \" policy=pz.MaxQuality(),\\n\",\n \" execution_strategy=\\\"parallel\\\",\\n\",\n \" max_workers=64,\\n\",\n \" progress=True,\\n\",\n \")\\n\",\n \"\\n\",\n \"output = plan.run(config=config, seed=SEED)\\n\",\n \"score_output(output, seed=SEED)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"ORUDS35-K1Nf\"\n },\n \"source\": [\n \"Now, let's run the program again while using our `train_datareader` as a validation dataset:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"a4htdaTDKpFc\"\n },\n \"outputs\": [],\n \"source\": [\n \"import logging\\n\",\n \"logger = logging.getLogger()\\n\",\n \"logger.disabled = True\\n\",\n \"\\n\",\n \"# execute pz plan\\n\",\n \"config = pz.QueryProcessorConfig(\\n\",\n \" policy=pz.MaxQuality(),\\n\",\n \" validator=validator,\\n\",\n \" optimizer_strategy=\\\"pareto\\\",\\n\",\n \" sentinel_execution_strategy=\\\"mab\\\",\\n\",\n \" execution_strategy=\\\"parallel\\\",\\n\",\n \" use_final_op_quality=True,\\n\",\n \" max_workers=64,\\n\",\n \" progress=True,\\n\",\n \")\\n\",\n \"\\n\",\n \"output = plan.run(config=config, k=6, j=4, sample_budget=72, seed=SEED)\\n\",\n \"score_output(output, seed=SEED)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"IT3iZr6-Kpab\"\n },\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"colab\": {\n \"provenance\": []\n },\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"name\": \"python\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 0\n}\n" }, { "path": "ruff.toml", "content": "# Config https://docs.astral.sh/ruff/configuration/\nline-length = 120\nindent-width = 4\nexclude = [\"*.ipynb\"]\n\n# Assume Python 3.8\ntarget-version = \"py38\"\n\n[lint]\nignore = [\"E501\"]\nfixable = [\"ALL\"]\nunfixable = []\nselect = [\"E\", \"F\", \"UP\", \"B\", \"SIM\", \"I\", \"N\"]\n" }, { "path": "scripts/capture_litellm_stats.py", "content": "#!/usr/bin/env python3\n\"\"\"\nScript to invoke LLM providers through LiteLLM and capture token/cost statistics.\n\nThis script:\n1. Loads messages from JSON files generated by generate_test_messages.py\n2. Sends requests through LiteLLM (the same path palimpzest uses)\n3. Saves all usage metadata and response stats returned by LiteLLM\n4. Waits 10 seconds\n5. Sends the request again and saves the second set of stats\n\nThis allows us to compare LiteLLM's reported statistics with:\n- Direct provider API calls (from capture_provider_stats.py)\n- Palimpzest's generator stats tracking\n\nSupported providers:\n- Anthropic: claude-sonnet-4-5-20250929 (text, image, text+image)\n- Google/Gemini: gemini-2.5-flash (all seven modality combinations)\n- OpenAI: gpt-4o-2024-08-06 (text, image, text+image)\n- OpenAI: gpt-4o-audio-preview (text+audio, audio)\n- Azure: gpt-4o via Azure OpenAI (text, image, text+image)\n\nOutput files are saved to: scripts/litellm_stats/\n\"\"\"\n\nimport argparse\nimport json\nimport os\nimport sys\nimport time\nimport uuid\nfrom typing import Any\n\nimport litellm\nfrom litellm.integrations.custom_logger import CustomLogger\n\n# Add project root to path\nsys.path.insert(0, os.path.join(os.path.dirname(__file__), \"..\"))\n\nimport contextlib\n\nfrom palimpzest.constants import Model\n\n\n# =============================================================================\n# RAW RESPONSE CAPTURE CALLBACK\n# =============================================================================\nclass RawProviderStatsCapture(CustomLogger):\n \"\"\"\n Custom LiteLLM callback to capture raw provider usage stats before normalization.\n\n LiteLLM normalizes all responses to OpenAI format, which loses provider-specific\n details like Gemini's per-modality token breakdowns. This callback captures the\n original provider response data.\n \"\"\"\n\n def __init__(self):\n self.last_raw_response = None\n self.last_raw_usage = None\n self.last_provider = None\n\n def log_success_event(self, kwargs, response_obj, start_time, end_time):\n \"\"\"Called after a successful LLM API call.\"\"\"\n try:\n # Store the provider info\n self.last_provider = kwargs.get(\"custom_llm_provider\") or kwargs.get(\"model\", \"\").split(\"/\")[0]\n\n # Try to get the original response from hidden params\n if hasattr(response_obj, \"_hidden_params\") and response_obj._hidden_params:\n hidden = response_obj._hidden_params\n self.last_raw_response = hidden.get(\"original_response\")\n\n # For some providers, the raw response might be in different locations\n if self.last_raw_response is None:\n self.last_raw_response = hidden.get(\"raw_response\")\n\n # Try to extract raw usage from the response object itself\n # Some providers have additional attributes that aren't in model_dump()\n if hasattr(response_obj, \"_response_ms\"):\n if self.last_raw_response is None:\n self.last_raw_response = {}\n self.last_raw_response[\"_response_ms\"] = response_obj._response_ms\n\n # For Vertex AI / Gemini, check for provider-specific usage fields\n if hasattr(response_obj, \"vertex_ai_usage_metadata\"):\n self.last_raw_usage = response_obj.vertex_ai_usage_metadata\n elif hasattr(response_obj, \"_vertex_ai_response\"):\n self.last_raw_response = response_obj._vertex_ai_response\n\n except Exception as e:\n # Don't let callback errors break the main flow\n print(f\" [Callback] Error capturing raw response: {e}\")\n\n def log_failure_event(self, kwargs, response_obj, start_time, end_time):\n \"\"\"Called after a failed LLM API call.\"\"\"\n self.last_raw_response = None\n self.last_raw_usage = None\n self.last_provider = None\n\n def reset(self):\n \"\"\"Reset captured data for next request.\"\"\"\n self.last_raw_response = None\n self.last_raw_usage = None\n self.last_provider = None\n\n def get_captured_data(self) -> dict[str, Any]:\n \"\"\"Return captured raw data and reset for next request.\"\"\"\n data = {\n \"raw_provider_response\": self.last_raw_response,\n \"raw_provider_usage\": self.last_raw_usage,\n \"detected_provider\": self.last_provider,\n }\n self.reset()\n return data\n\n\n# Global callback instance\nraw_stats_capture = RawProviderStatsCapture()\n\n# Register the callback with LiteLLM\nlitellm.callbacks = [raw_stats_capture]\n\n# Enable return of response headers (helps with some providers)\nlitellm.return_response_headers = True\n\n\n# =============================================================================\n# PROVIDER CONFIGURATIONS\n# =============================================================================\n# Maps provider name to Model enum and supported modalities\n# The Model enum is used for:\n# 1. Getting the LiteLLM model name via model.value\n# 2. Initializing PromptManager which needs a Model enum\nPROVIDER_MODALITY_SUPPORT = {\n \"anthropic\": {\n \"model\": Model.CLAUDE_4_5_SONNET,\n \"supported_modalities\": [\"text-only\", \"image-only\", \"text-image\"],\n # Note: Anthropic does not support audio\n },\n \"openai\": {\n \"model\": Model.GPT_4o,\n \"supported_modalities\": [\"text-only\", \"image-only\", \"text-image\"],\n },\n \"openai-audio\": {\n \"model\": Model.GPT_4o_AUDIO_PREVIEW,\n \"supported_modalities\": [\"audio-only\", \"text-audio\"],\n },\n \"gemini\": {\n \"model\": Model.GOOGLE_GEMINI_2_5_FLASH,\n \"supported_modalities\": [\n \"text-only\",\n \"image-only\",\n \"audio-only\",\n \"text-image\",\n \"text-audio\",\n \"image-audio\",\n \"text-image-audio\",\n ],\n },\n \"vertex_ai\": {\n \"model\": Model.GEMINI_2_5_FLASH,\n \"supported_modalities\": [\n \"text-only\",\n \"image-only\",\n \"audio-only\",\n \"text-image\",\n \"text-audio\",\n \"image-audio\",\n \"text-image-audio\",\n ],\n },\n \"azure\": {\n \"model\": Model.AZURE_GPT_4o,\n \"supported_modalities\": [\"text-only\", \"image-only\", \"text-image\"],\n },\n}\n\n\ndef load_messages(modality: str, provider: str, messages_dir: str) -> list[dict]:\n \"\"\"Load messages from JSON file for a given modality/provider combination.\"\"\"\n filepath = os.path.join(messages_dir, f\"{modality}_{provider}.json\")\n with open(filepath) as f:\n return json.load(f)\n\n\ndef transform_messages_for_litellm(messages: list[dict]) -> list[dict]:\n \"\"\"\n Transform palimpzest message format to LiteLLM-compatible format.\n\n LiteLLM expects:\n - Messages with role and content\n - Content can be string or list of content blocks\n - No 'type' field at the message level (that's palimpzest-specific)\n\n This function consolidates multiple user messages with different types\n into single messages with combined content.\n \"\"\"\n litellm_messages = []\n\n for msg in messages:\n role = msg.get(\"role\")\n msg_type = msg.get(\"type\")\n content = msg.get(\"content\")\n\n if role == \"system\":\n # System messages pass through as-is\n # Content may be string or list of content blocks (for caching)\n litellm_messages.append({\"role\": \"system\", \"content\": content})\n\n elif role == \"user\":\n # User messages need consolidation\n if msg_type == \"text\":\n # Text content - string or list of content blocks\n if litellm_messages and litellm_messages[-1][\"role\"] == \"user\":\n # Merge with existing user message\n existing = litellm_messages[-1][\"content\"]\n if isinstance(existing, str):\n if isinstance(content, str):\n litellm_messages[-1][\"content\"] = [\n {\"type\": \"text\", \"text\": existing},\n {\"type\": \"text\", \"text\": content},\n ]\n else:\n litellm_messages[-1][\"content\"] = [\n {\"type\": \"text\", \"text\": existing}\n ] + content\n else:\n if isinstance(content, str):\n existing.append({\"type\": \"text\", \"text\": content})\n else:\n existing.extend(content)\n else:\n litellm_messages.append({\"role\": \"user\", \"content\": content})\n\n elif msg_type == \"image\":\n # Image content - list of image_url blocks\n if litellm_messages and litellm_messages[-1][\"role\"] == \"user\":\n existing = litellm_messages[-1][\"content\"]\n if isinstance(existing, str):\n litellm_messages[-1][\"content\"] = [\n {\"type\": \"text\", \"text\": existing}\n ] + content\n else:\n existing.extend(content)\n else:\n litellm_messages.append({\"role\": \"user\", \"content\": content})\n\n elif msg_type == \"input_audio\":\n # Audio content - list of input_audio blocks\n if litellm_messages and litellm_messages[-1][\"role\"] == \"user\":\n existing = litellm_messages[-1][\"content\"]\n if isinstance(existing, str):\n litellm_messages[-1][\"content\"] = [\n {\"type\": \"text\", \"text\": existing}\n ] + content\n else:\n existing.extend(content)\n else:\n litellm_messages.append({\"role\": \"user\", \"content\": content})\n\n elif role == \"assistant\":\n litellm_messages.append({\"role\": \"assistant\", \"content\": content})\n\n return litellm_messages\n\n\ndef call_litellm_api(\n messages: list[dict],\n model: Model,\n provider: str,\n cache_key: str | None = None,\n) -> dict[str, Any]:\n \"\"\"\n Call LiteLLM completion API and return all usage statistics.\n\n This function captures both:\n - Option A: Raw provider usage via callback (if available)\n - Option C: Normalized LiteLLM usage (fallback)\n\n Args:\n messages: List of message dicts (palimpzest format)\n model: Model enum (used for model name and provider detection)\n provider: Provider name for logging\n cache_key: Optional prompt_cache_key for OpenAI sticky routing to same cache shard\n\n Returns dict with:\n - usage: Normalized usage dict from LiteLLM response (Option C fallback)\n - usage_raw: Raw provider usage if captured via callback (Option A)\n - response_content: First 200 chars of response\n - model: Model used\n - raw_response: Full response object serialized\n \"\"\"\n # Reset the callback to capture fresh data for this request\n raw_stats_capture.reset()\n\n # Transform messages to LiteLLM format\n litellm_messages = transform_messages_for_litellm(messages)\n\n # Get the LiteLLM model name from the Model enum\n model_name = model.value\n\n # Set up completion kwargs\n completion_kwargs = {\n \"temperature\": 0.0,\n }\n\n # Add modalities for audio models\n if \"audio\" in model_name.lower():\n completion_kwargs[\"modalities\"] = [\"text\"]\n\n # Apply provider-specific caching configuration\n # Messages from generator_messages already have cache_control markers for Anthropic\n if (model.is_provider_openai() or model.is_provider_azure()) and cache_key:\n # OpenAI: Use prompt_cache_key for sticky routing to the same cache shard\n # https://platform.openai.com/docs/guides/prompt-caching\n completion_kwargs[\"extra_body\"] = {\"prompt_cache_key\": cache_key}\n\n # Make the LiteLLM call\n response = litellm.completion(\n model=model_name,\n messages=litellm_messages,\n **completion_kwargs,\n )\n\n # ==========================================================================\n # Option C (Fallback): Extract normalized usage stats from LiteLLM response\n # ==========================================================================\n usage_normalized = {}\n if response.usage:\n usage_normalized = response.usage.model_dump()\n\n # ==========================================================================\n # Option A: Get raw provider data captured by callback\n # ==========================================================================\n callback_data = raw_stats_capture.get_captured_data()\n usage_raw = callback_data.get(\"raw_provider_usage\")\n\n # Also try to extract raw usage from _hidden_params\n hidden_params = {}\n try:\n if hasattr(response, \"_hidden_params\") and response._hidden_params:\n hidden_params = dict(response._hidden_params)\n # Some providers store original response here\n if \"original_response\" in hidden_params:\n original = hidden_params[\"original_response\"]\n if isinstance(original, dict) and \"usage_metadata\" in original:\n usage_raw = original[\"usage_metadata\"]\n elif hasattr(original, \"usage_metadata\"):\n with contextlib.suppress(Exception):\n usage_raw = original.usage_metadata.model_dump() if hasattr(original.usage_metadata, \"model_dump\") else dict(original.usage_metadata)\n except Exception:\n pass\n\n # Get response text safely\n try:\n response_text = (\n response.choices[0].message.content[:200]\n if response.choices and response.choices[0].message.content\n else None\n )\n except Exception:\n response_text = None\n\n # Serialize the full response for debugging\n try:\n raw_response = response.model_dump()\n except Exception:\n raw_response = str(response)\n\n return {\n \"provider\": provider,\n \"model\": model_name,\n \"usage\": usage_normalized, # Option C: Normalized LiteLLM format\n \"usage_raw\": usage_raw, # Option A: Raw provider format (if captured)\n \"response_content\": response_text,\n \"raw_response\": raw_response,\n \"hidden_params\": hidden_params,\n \"callback_data\": callback_data,\n }\n\n\ndef capture_stats_for_provider(\n provider: str,\n modality: str,\n messages: list[dict],\n model: Model,\n) -> dict[str, Any]:\n \"\"\"\n Capture stats for a provider by making two requests with a delay.\n\n Args:\n provider: Provider name (for logging and file naming)\n modality: Modality name\n messages: List of message dicts\n model: Model enum\n\n Returns dict with:\n - first_request: stats from first request\n - second_request: stats from second request (should show cache hits)\n \"\"\"\n # Generate a unique cache key for OpenAI (ensures both requests hit the same cache shard)\n # Reference: capture_provider_stats.py and PromptManager.__init__\n openai_cache_key = f\"pz-test-{uuid.uuid4().hex[:12]}\" if provider in (\"openai\", \"openai-audio\", \"azure\") else None\n\n print(\" First request...\")\n first_stats = call_litellm_api(messages, model, provider, cache_key=openai_cache_key)\n print(f\" Usage: {first_stats['usage']}\")\n\n print(\" Waiting 20 seconds for cache to be available...\")\n time.sleep(20)\n\n print(\" Second request (should show cache hits)...\")\n second_stats = call_litellm_api(messages, model, provider, cache_key=openai_cache_key)\n print(f\" Usage: {second_stats['usage']}\")\n\n return {\n \"provider\": provider,\n \"model\": model.value,\n \"modality\": modality,\n \"first_request\": first_stats,\n \"second_request\": second_stats,\n }\n\n\ndef save_stats(stats: dict[str, Any], output_dir: str, provider: str, modality: str) -> str:\n \"\"\"Save stats to a JSON file.\"\"\"\n os.makedirs(output_dir, exist_ok=True)\n output_path = os.path.join(output_dir, f\"{provider}_{modality}.json\")\n\n with open(output_path, \"w\") as f:\n json.dump(stats, f, indent=2, default=str)\n\n return output_path\n\n\ndef main():\n \"\"\"Capture LiteLLM stats for supported provider/modality combinations.\"\"\"\n parser = argparse.ArgumentParser(\n description=\"Capture token/cost statistics from LLM providers via LiteLLM.\",\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=f\"\"\"\nAvailable providers: {', '.join(PROVIDER_MODALITY_SUPPORT.keys())}\nAvailable modalities: text-only, image-only, audio-only, text-image, text-audio, image-audio, text-image-audio\n\nExamples:\n python capture_litellm_stats.py # Run all providers/modalities\n python capture_litellm_stats.py -p openai # Run all modalities for OpenAI\n python capture_litellm_stats.py -p openai -m text-only # Run only text-only for OpenAI\n python capture_litellm_stats.py -p anthropic -m text-image # Run only text-image for Anthropic\n \"\"\"\n )\n parser.add_argument(\n \"-p\", \"--provider\",\n nargs=\"+\",\n choices=list(PROVIDER_MODALITY_SUPPORT.keys()),\n help=\"Provider(s) to run. If not specified, runs all providers.\",\n )\n parser.add_argument(\n \"-m\", \"--modality\",\n nargs=\"+\",\n choices=[\"text-only\", \"image-only\", \"audio-only\", \"text-image\", \"text-audio\", \"image-audio\", \"text-image-audio\"],\n help=\"Modality(ies) to run. If not specified, runs all supported modalities for each provider.\",\n )\n args = parser.parse_args()\n\n messages_dir = os.path.join(\n os.path.dirname(__file__),\n \"..\",\n \"tests\",\n \"pytest\",\n \"data\",\n \"generator_messages\",\n )\n messages_dir = os.path.abspath(messages_dir)\n\n output_dir = os.path.join(\n os.path.dirname(__file__),\n \"litellm_stats\",\n )\n output_dir = os.path.abspath(output_dir)\n\n print(f\"Loading messages from: {messages_dir}\")\n print(f\"Saving stats to: {output_dir}\\n\")\n\n # Ensure output directory exists\n os.makedirs(output_dir, exist_ok=True)\n\n # Determine which providers to run\n providers_to_run = args.provider if args.provider else list(PROVIDER_MODALITY_SUPPORT.keys())\n print(f\"Providers to run: {providers_to_run}\\n\")\n\n for provider in providers_to_run:\n config = PROVIDER_MODALITY_SUPPORT[provider]\n model = config[\"model\"]\n supported_modalities = config[\"supported_modalities\"]\n\n # Filter modalities if specified\n if args.modality:\n modalities_to_run = [m for m in args.modality if m in supported_modalities]\n if not modalities_to_run:\n print(f\"\\nProvider: {provider} - SKIPPED (none of {args.modality} supported)\")\n continue\n else:\n modalities_to_run = supported_modalities\n\n print(f\"\\nProvider: {provider} (model: {model.value})\")\n print(f\" Modalities to run: {modalities_to_run}\")\n\n for modality in modalities_to_run:\n print(f\"\\n Processing modality: {modality}\")\n\n try:\n messages = load_messages(modality, provider, messages_dir)\n print(f\" Loaded {len(messages)} messages from {modality}_{provider}.json\")\n\n stats = capture_stats_for_provider(provider, modality, messages, model)\n\n output_path = save_stats(stats, output_dir, provider, modality)\n print(f\" Saved to: {output_path}\")\n\n except FileNotFoundError as e:\n print(f\" SKIPPED: Message file not found - {e}\")\n except Exception as e:\n print(f\" ERROR: {e}\")\n import traceback\n traceback.print_exc()\n\n print(\"\\nDone!\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "scripts/capture_provider_stats.py", "content": "#!/usr/bin/env python3\n\"\"\"\nScript to directly invoke LLM providers and capture token/cost statistics.\n\nThis script:\n1. Loads messages from JSON files generated by generate_test_messages.py\n2. Sends requests directly to each provider's API (not through litellm)\n3. Saves the token/cost related stats returned by the provider\n4. Waits 10 seconds\n5. Sends the request again and saves the second set of stats\n\nThis allows us to establish baseline expectations for what the providers return,\nwhich can then be used to validate the palimpzest generator's stats tracking.\n\nSupported providers:\n- Anthropic: claude-sonnet-4-5-20250929 (text, image, text+image)\n- Google/Vertex AI: gemini-2.5-flash (all seven modality combinations)\n- OpenAI: gpt-4o-2024-08-06 (text, image, text+image)\n- OpenAI: gpt-4o-audio-preview (text+audio, audio)\n- Azure: gpt-4o-2024-08-06 via Azure OpenAI (text, image, text+image)\n\nOutput files are saved to: tests/pytest/scripts/provider_stats/\n\"\"\"\n\nimport argparse\nimport base64\nimport json\nimport os\nimport sys\nimport time\nimport uuid\nfrom typing import Any\n\n\ndef detect_image_media_type(base64_data: str) -> str:\n \"\"\"\n Detect the actual image format from base64 data by examining the magic bytes.\n\n Args:\n base64_data: Base64-encoded image data.\n\n Returns:\n The detected media type (e.g., 'image/png', 'image/jpeg').\n Defaults to 'image/jpeg' if format cannot be determined.\n \"\"\"\n try:\n # Decode first few bytes to check magic numbers\n header = base64.b64decode(base64_data[:32])\n\n # PNG: 89 50 4E 47 0D 0A 1A 0A\n if header[:8] == b\"\\x89PNG\\r\\n\\x1a\\n\":\n return \"image/png\"\n # JPEG: FF D8 FF\n if header[:3] == b\"\\xff\\xd8\\xff\":\n return \"image/jpeg\"\n # GIF: GIF87a or GIF89a\n if header[:6] in (b\"GIF87a\", b\"GIF89a\"):\n return \"image/gif\"\n # WebP: RIFF....WEBP\n if header[:4] == b\"RIFF\" and header[8:12] == b\"WEBP\":\n return \"image/webp\"\n except Exception:\n pass\n\n return \"image/jpeg\"\n\n# Add project root to path\nsys.path.insert(0, os.path.join(os.path.dirname(__file__), \"..\"))\n\n\n# =============================================================================\n# PROVIDER CONFIGURATIONS\n# =============================================================================\nPROVIDER_MODALITY_SUPPORT = {\n \"anthropic\": {\n \"model\": \"claude-sonnet-4-5-20250929\",\n \"supported_modalities\": [\"text-only\", \"image-only\", \"text-image\"],\n # Note: Anthropic does not support audio\n },\n \"openai\": {\n \"model\": \"gpt-4o-2024-08-06\",\n \"supported_modalities\": [\"text-only\", \"image-only\", \"text-image\"],\n },\n \"openai-audio\": {\n \"model\": \"gpt-4o-audio-preview\",\n \"supported_modalities\": [\"audio-only\", \"text-audio\"],\n },\n \"gemini\": {\n \"model\": \"gemini-2.5-flash\",\n \"supported_modalities\": [\n \"text-only\",\n \"image-only\",\n \"audio-only\",\n \"text-image\",\n \"text-audio\",\n \"image-audio\",\n \"text-image-audio\",\n ],\n },\n \"vertex_ai\": {\n \"model\": \"gemini-2.5-flash\",\n \"supported_modalities\": [\n \"text-only\",\n \"image-only\",\n \"audio-only\",\n \"text-image\",\n \"text-audio\",\n \"image-audio\",\n \"text-image-audio\",\n ],\n },\n \"azure\": {\n \"model\": \"gpt-4o-2024-08-06\",\n \"supported_modalities\": [\"text-only\", \"image-only\", \"text-image\"],\n },\n}\n\n\ndef load_messages(modality: str, provider: str, messages_dir: str) -> list[dict]:\n \"\"\"Load messages from JSON file for a given modality/provider combination.\"\"\"\n filepath = os.path.join(messages_dir, f\"{modality}_{provider}.json\")\n with open(filepath) as f:\n return json.load(f)\n\n\ndef transform_messages_for_openai(messages: list[dict]) -> list[dict]:\n \"\"\"\n Transform palimpzest/litellm message format to OpenAI API format.\n\n OpenAI expects:\n - system messages with string content\n - user messages with string content or array of content parts\n\n Input messages may have content as string or list of content blocks.\n \"\"\"\n openai_messages = []\n\n for msg in messages:\n role = msg.get(\"role\")\n msg_type = msg.get(\"type\")\n content = msg.get(\"content\")\n\n if role == \"system\":\n # System content may be string or list of content blocks\n if isinstance(content, list):\n # Extract text from content blocks\n text_parts = [block.get(\"text\", \"\") for block in content if block.get(\"type\") == \"text\"]\n openai_messages.append({\"role\": \"system\", \"content\": \"\".join(text_parts)})\n else:\n openai_messages.append({\"role\": \"system\", \"content\": content})\n\n elif role == \"user\":\n if msg_type == \"text\":\n # Content may be string or list of content blocks\n if isinstance(content, list):\n # Already content blocks - add them directly\n content_parts = []\n for block in content:\n # Convert to OpenAI format (remove cache_control if present)\n openai_block = {\"type\": block.get(\"type\", \"text\")}\n if block.get(\"type\") == \"text\":\n openai_block[\"text\"] = block.get(\"text\", \"\")\n content_parts.append(openai_block)\n\n if openai_messages and openai_messages[-1][\"role\"] == \"user\":\n existing_content = openai_messages[-1][\"content\"]\n if isinstance(existing_content, str):\n openai_messages[-1][\"content\"] = [\n {\"type\": \"text\", \"text\": existing_content}\n ] + content_parts\n else:\n existing_content.extend(content_parts)\n else:\n openai_messages.append({\"role\": \"user\", \"content\": content_parts})\n else:\n # String content\n if openai_messages and openai_messages[-1][\"role\"] == \"user\":\n existing_content = openai_messages[-1][\"content\"]\n if isinstance(existing_content, str):\n openai_messages[-1][\"content\"] = [\n {\"type\": \"text\", \"text\": existing_content},\n {\"type\": \"text\", \"text\": content},\n ]\n else:\n existing_content.append({\"type\": \"text\", \"text\": content})\n else:\n openai_messages.append({\"role\": \"user\", \"content\": content})\n\n elif msg_type == \"image\":\n # Image content\n image_parts = []\n for img in content:\n if img.get(\"type\") == \"image_url\":\n image_parts.append(img)\n\n if openai_messages and openai_messages[-1][\"role\"] == \"user\":\n existing_content = openai_messages[-1][\"content\"]\n if isinstance(existing_content, str):\n openai_messages[-1][\"content\"] = [\n {\"type\": \"text\", \"text\": existing_content}\n ] + image_parts\n else:\n existing_content.extend(image_parts)\n else:\n openai_messages.append({\"role\": \"user\", \"content\": image_parts})\n\n elif msg_type == \"input_audio\":\n # Audio content\n audio_parts = []\n for audio in content:\n if audio.get(\"type\") == \"input_audio\":\n audio_parts.append(audio)\n\n if openai_messages and openai_messages[-1][\"role\"] == \"user\":\n existing_content = openai_messages[-1][\"content\"]\n if isinstance(existing_content, str):\n openai_messages[-1][\"content\"] = [\n {\"type\": \"text\", \"text\": existing_content}\n ] + audio_parts\n else:\n existing_content.extend(audio_parts)\n else:\n openai_messages.append({\"role\": \"user\", \"content\": audio_parts})\n\n return openai_messages\n\n\ndef transform_messages_for_anthropic(messages: list[dict]) -> tuple[str | None, list[dict]]:\n \"\"\"\n Transform palimpzest/litellm message format to Anthropic API format.\n\n Input messages may already have cache_control markers from PromptManager.\n This function preserves those markers while converting to Anthropic's native format.\n\n Anthropic expects:\n - system as a separate parameter (not in messages)\n - user/assistant messages with content as array of content blocks\n - cache_control markers for caching (preserved from input)\n \"\"\"\n system_prompt = None\n anthropic_messages = []\n\n for msg in messages:\n role = msg.get(\"role\")\n msg_type = msg.get(\"type\")\n content = msg.get(\"content\")\n\n if role == \"system\":\n # Anthropic uses system as a separate parameter\n # Content may already be a list of content blocks with cache_control\n if isinstance(content, list):\n # Already in content block format (from PromptManager)\n system_prompt = content\n else:\n # String content - wrap in content block with cache_control\n system_prompt = [\n {\n \"type\": \"text\",\n \"text\": content,\n \"cache_control\": {\"type\": \"ephemeral\"},\n }\n ]\n\n elif role == \"user\":\n if msg_type == \"text\":\n # Content may be string or list of content blocks\n if isinstance(content, list):\n # Already content blocks (may have cache_control) - preserve them\n for block in content:\n if anthropic_messages and anthropic_messages[-1][\"role\"] == \"user\":\n anthropic_messages[-1][\"content\"].append(block)\n else:\n anthropic_messages.append({\"role\": \"user\", \"content\": [block]})\n else:\n # String content\n content_block = {\"type\": \"text\", \"text\": content}\n if anthropic_messages and anthropic_messages[-1][\"role\"] == \"user\":\n anthropic_messages[-1][\"content\"].append(content_block)\n else:\n anthropic_messages.append({\"role\": \"user\", \"content\": [content_block]})\n\n elif msg_type == \"image\":\n # Image content - Anthropic uses base64 format\n for img in content:\n if img.get(\"type\") == \"image_url\":\n url = img[\"image_url\"][\"url\"]\n if url.startswith(\"data:\"):\n # Extract base64 data\n _, data = url.split(\";base64,\")\n # Detect actual media type from image data (in case URL has wrong type)\n media_type = detect_image_media_type(data)\n image_block = {\n \"type\": \"image\",\n \"source\": {\n \"type\": \"base64\",\n \"media_type\": media_type,\n \"data\": data,\n },\n }\n # Preserve cache_control if present on the original block\n if \"cache_control\" in img:\n image_block[\"cache_control\"] = img[\"cache_control\"]\n if anthropic_messages and anthropic_messages[-1][\"role\"] == \"user\":\n anthropic_messages[-1][\"content\"].append(image_block)\n else:\n anthropic_messages.append({\"role\": \"user\", \"content\": [image_block]})\n\n return system_prompt, anthropic_messages\n\n\ndef transform_messages_for_gemini(messages: list[dict]) -> tuple[str | None, list[dict]]:\n \"\"\"\n Transform palimpzest/litellm message format to Gemini API format.\n\n Gemini expects:\n - role: \"user\" or \"model\"\n - parts: list of content parts\n\n Input messages may have content as string or list of content blocks.\n \"\"\"\n gemini_contents = []\n system_instruction = None\n\n for msg in messages:\n role = msg.get(\"role\")\n msg_type = msg.get(\"type\")\n content = msg.get(\"content\")\n\n if role == \"system\":\n # Gemini uses system_instruction\n # Content may be string or list of content blocks\n if isinstance(content, list):\n # Extract text from content blocks\n text_parts = [block.get(\"text\", \"\") for block in content if block.get(\"type\") == \"text\"]\n system_instruction = \"\".join(text_parts)\n else:\n system_instruction = content\n\n elif role == \"user\":\n parts = []\n\n if msg_type == \"text\":\n # Content may be string or list of content blocks\n if isinstance(content, list):\n for block in content:\n if block.get(\"type\") == \"text\":\n parts.append({\"text\": block.get(\"text\", \"\")})\n else:\n parts.append({\"text\": content})\n\n elif msg_type == \"image\":\n for img in content:\n if img.get(\"type\") == \"image_url\":\n url = img[\"image_url\"][\"url\"]\n if url.startswith(\"data:\"):\n _, data = url.split(\";base64,\")\n # Detect actual media type from image data\n media_type = detect_image_media_type(data)\n parts.append({\n \"inline_data\": {\n \"mime_type\": media_type,\n \"data\": data,\n }\n })\n\n elif msg_type == \"input_audio\":\n for audio in content:\n if audio.get(\"type\") == \"input_audio\":\n audio_data = audio[\"input_audio\"]\n parts.append({\n \"inline_data\": {\n \"mime_type\": f\"audio/{audio_data.get('format', 'wav')}\",\n \"data\": audio_data[\"data\"],\n }\n })\n\n if parts:\n if gemini_contents and gemini_contents[-1][\"role\"] == \"user\":\n gemini_contents[-1][\"parts\"].extend(parts)\n else:\n gemini_contents.append({\"role\": \"user\", \"parts\": parts})\n\n return system_instruction, gemini_contents\n\n\ndef call_openai_api(messages: list[dict], model: str, cache_key: str | None = None) -> dict[str, Any]:\n \"\"\"\n Call OpenAI API directly and return usage statistics.\n\n Args:\n messages: List of message dicts\n model: Model name\n cache_key: Optional prompt_cache_key for sticky routing to same cache shard\n\n Returns dict with:\n - completion_tokens\n - prompt_tokens\n - prompt_tokens_details (cached_tokens, text_tokens, image_tokens, audio_tokens)\n - total_tokens\n \"\"\"\n import openai\n\n client = openai.OpenAI()\n\n openai_messages = transform_messages_for_openai(messages)\n\n kwargs = {\"model\": model, \"messages\": openai_messages, \"temperature\": 0.0}\n\n # Check if this is an audio model\n if \"audio\" in model:\n kwargs[\"modalities\"] = [\"text\"]\n\n # Add prompt_cache_key for caching (ensures requests route to same cache shard)\n if cache_key:\n kwargs[\"extra_body\"] = {\"prompt_cache_key\": cache_key}\n\n response = client.chat.completions.create(**kwargs)\n\n # Extract complete usage stats\n usage_dict = {}\n if response.usage:\n usage_dict = response.usage.model_dump()\n\n # Get response text safely\n try:\n response_text = response.choices[0].message.content[:200] if response.choices and response.choices[0].message.content else None\n except Exception:\n response_text = None\n\n # Serialize the full response\n try:\n raw_response = response.model_dump()\n except Exception:\n raw_response = str(response)\n\n return {\n \"provider\": \"openai\",\n \"model\": model,\n \"usage\": usage_dict,\n \"response_content\": response_text,\n \"raw_response\": raw_response,\n }\n\n\n# NOTE: this function was generated speculatively and has not been tested, so it may have errors\ndef call_azure_api(messages: list[dict], model: str, cache_key: str | None = None) -> dict[str, Any]:\n \"\"\"\n Call Azure OpenAI API directly and return usage statistics.\n\n Uses the same message format as OpenAI, but routes through Azure endpoints.\n\n Args:\n messages: List of message dicts\n model: Model name (deployment name)\n cache_key: Optional prompt_cache_key for sticky routing to same cache shard\n\n Returns dict with:\n - completion_tokens\n - prompt_tokens\n - prompt_tokens_details (cached_tokens, text_tokens, image_tokens, audio_tokens)\n - total_tokens\n \"\"\"\n import openai\n\n api_key = os.environ.get(\"AZURE_API_KEY\") or os.environ.get(\"AZURE_OPENAI_API_KEY\")\n azure_endpoint = os.environ.get(\"AZURE_API_BASE\")\n api_version = os.environ.get(\"AZURE_API_VERSION\", \"2024-12-01-preview\")\n\n if not api_key:\n raise ValueError(\"AZURE_API_KEY or AZURE_OPENAI_API_KEY must be set\")\n if not azure_endpoint:\n raise ValueError(\"AZURE_API_BASE must be set\")\n\n client = openai.AzureOpenAI(\n api_key=api_key,\n azure_endpoint=azure_endpoint,\n api_version=api_version,\n )\n\n openai_messages = transform_messages_for_openai(messages)\n\n kwargs = {\"model\": model, \"messages\": openai_messages, \"temperature\": 0.0}\n\n # Add prompt_cache_key for caching (ensures requests route to same cache shard)\n if cache_key:\n kwargs[\"extra_body\"] = {\"prompt_cache_key\": cache_key}\n\n response = client.chat.completions.create(**kwargs)\n\n # Extract complete usage stats\n usage_dict = {}\n if response.usage:\n usage_dict = response.usage.model_dump()\n\n # Get response text safely\n try:\n response_text = response.choices[0].message.content[:200] if response.choices and response.choices[0].message.content else None\n except Exception:\n response_text = None\n\n # Serialize the full response\n try:\n raw_response = response.model_dump()\n except Exception:\n raw_response = str(response)\n\n return {\n \"provider\": \"azure\",\n \"model\": model,\n \"usage\": usage_dict,\n \"response_content\": response_text,\n \"raw_response\": raw_response,\n }\n\n\ndef call_anthropic_api(messages: list[dict], model: str) -> dict[str, Any]:\n \"\"\"\n Call Anthropic API directly and return usage statistics.\n\n Returns dict with:\n - input_tokens\n - output_tokens\n - cache_creation_input_tokens\n - cache_read_input_tokens\n \"\"\"\n import anthropic\n\n client = anthropic.Anthropic()\n\n system_prompt, anthropic_messages = transform_messages_for_anthropic(messages)\n\n response = client.messages.create(\n model=model,\n max_tokens=1024,\n system=system_prompt,\n messages=anthropic_messages,\n )\n\n # Extract complete usage stats\n usage_dict = {}\n if response.usage:\n usage_dict = response.usage.model_dump()\n\n # Get response text safely\n try:\n response_text = response.content[0].text[:200] if response.content and response.content[0].text else None\n except Exception:\n response_text = None\n\n # Serialize the full response\n try:\n raw_response = response.model_dump()\n except Exception:\n raw_response = str(response)\n\n return {\n \"provider\": \"anthropic\",\n \"model\": model,\n \"usage\": usage_dict,\n \"response_content\": response_text,\n \"raw_response\": raw_response,\n }\n\n\ndef call_gemini_api(messages: list[dict], model: str, use_vertex: bool = False) -> dict[str, Any]:\n \"\"\"\n Call Gemini API directly and return usage statistics.\n\n Args:\n messages: List of message dicts\n model: Model name\n use_vertex: If True, use Vertex AI; otherwise use Google AI Studio\n\n Returns dict with usage statistics.\n \"\"\"\n from google import genai\n from google.genai import types\n\n system_instruction, gemini_contents = transform_messages_for_gemini(messages)\n\n # Create client for Google AI Studio or Vertex AI\n if use_vertex:\n # Vertex AI requires project and location\n import os\n client = genai.Client(\n vertexai=True,\n project=os.environ.get(\"GOOGLE_CLOUD_PROJECT\", os.environ.get(\"VERTEXAI_PROJECT\")),\n location=os.environ.get(\"GOOGLE_CLOUD_LOCATION\", os.environ.get(\"VERTEXAI_LOCATION\", \"us-central1\")),\n )\n else:\n # Google AI Studio uses API key from environment\n client = genai.Client()\n\n # Build the config\n config = types.GenerateContentConfig(\n temperature=0.0,\n system_instruction=system_instruction if system_instruction else None,\n )\n\n response = client.models.generate_content(\n model=model,\n contents=gemini_contents,\n config=config,\n )\n\n # Extract complete usage stats from usage_metadata\n usage_metadata = response.usage_metadata\n usage_dict = {}\n if usage_metadata:\n # Try model_dump() first (Pydantic models), then to_dict(), then manual extraction\n try:\n usage_dict = usage_metadata.model_dump()\n except AttributeError:\n try:\n usage_dict = usage_metadata.to_dict()\n except AttributeError:\n # Manual extraction of known Gemini usage fields\n usage_dict = {\n \"prompt_token_count\": getattr(usage_metadata, \"prompt_token_count\", None),\n \"candidates_token_count\": getattr(usage_metadata, \"candidates_token_count\", None),\n \"total_token_count\": getattr(usage_metadata, \"total_token_count\", None),\n \"cached_content_token_count\": getattr(usage_metadata, \"cached_content_token_count\", None),\n }\n\n # Get response text safely\n try:\n response_text = response.text[:200] if response.text else None\n except Exception:\n response_text = None\n\n # Serialize the full response\n try:\n # Try model_dump() first (Pydantic models)\n raw_response = response.model_dump()\n except AttributeError:\n try:\n raw_response = response.to_dict()\n except AttributeError:\n # Manual serialization\n try:\n raw_response = {\n \"text\": response.text if hasattr(response, \"text\") else None,\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [{\"text\": getattr(part, \"text\", str(part))} for part in c.content.parts] if c.content and c.content.parts else [],\n \"role\": c.content.role if c.content else None,\n },\n \"finish_reason\": str(c.finish_reason) if hasattr(c, \"finish_reason\") else None,\n }\n for c in (response.candidates or [])\n ],\n \"usage_metadata\": usage_dict,\n \"model_version\": getattr(response, \"model_version\", None),\n }\n except Exception as e:\n raw_response = {\"error\": str(e), \"response_str\": str(response)}\n\n return {\n \"provider\": \"vertex_ai\" if use_vertex else \"gemini\",\n \"model\": model,\n \"usage\": usage_dict,\n \"response_content\": response_text,\n \"raw_response\": raw_response,\n }\n\n\ndef capture_stats_for_provider(\n provider: str,\n modality: str,\n messages: list[dict],\n model: str,\n) -> dict[str, Any]:\n \"\"\"\n Capture stats for a provider by making two requests with a 10-second gap.\n\n Returns dict with:\n - first_request: stats from first request\n - second_request: stats from second request (should show cache hits)\n \"\"\"\n # Generate a unique cache key for OpenAI/Azure (ensures both requests hit the same cache shard)\n openai_cache_key = f\"pz-test-{uuid.uuid4().hex[:12]}\" if provider in (\"openai\", \"openai-audio\", \"azure\") else None\n\n print(\" First request...\")\n if provider == \"openai\" or provider == \"openai-audio\":\n first_stats = call_openai_api(messages, model, cache_key=openai_cache_key)\n elif provider == \"azure\":\n first_stats = call_azure_api(messages, model, cache_key=openai_cache_key)\n elif provider == \"anthropic\":\n first_stats = call_anthropic_api(messages, model)\n elif provider == \"gemini\":\n first_stats = call_gemini_api(messages, model, use_vertex=False)\n elif provider == \"vertex_ai\":\n first_stats = call_gemini_api(messages, model, use_vertex=True)\n else:\n raise ValueError(f\"Unknown provider: {provider}\")\n\n print(f\" Usage: {first_stats['usage']}\")\n\n print(\" Waiting 20 seconds for cache to be available...\")\n time.sleep(20)\n\n print(\" Second request (should show cache hits)...\")\n if provider == \"openai\" or provider == \"openai-audio\":\n second_stats = call_openai_api(messages, model, cache_key=openai_cache_key)\n elif provider == \"azure\":\n second_stats = call_azure_api(messages, model, cache_key=openai_cache_key)\n elif provider == \"anthropic\":\n second_stats = call_anthropic_api(messages, model)\n elif provider == \"gemini\":\n second_stats = call_gemini_api(messages, model, use_vertex=False)\n elif provider == \"vertex_ai\":\n second_stats = call_gemini_api(messages, model, use_vertex=True)\n\n print(f\" Usage: {second_stats['usage']}\")\n\n return {\n \"provider\": provider,\n \"model\": model,\n \"modality\": modality,\n \"first_request\": first_stats,\n \"second_request\": second_stats,\n }\n\n\ndef save_stats(stats: dict[str, Any], output_dir: str, provider: str, modality: str) -> str:\n \"\"\"Save stats to a JSON file.\"\"\"\n os.makedirs(output_dir, exist_ok=True)\n output_path = os.path.join(output_dir, f\"{provider}_{modality}.json\")\n\n with open(output_path, \"w\") as f:\n json.dump(stats, f, indent=2)\n\n return output_path\n\n\ndef main():\n \"\"\"Capture provider stats for supported provider/modality combinations.\"\"\"\n parser = argparse.ArgumentParser(\n description=\"Capture token/cost statistics from LLM providers.\",\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=f\"\"\"\nAvailable providers: {', '.join(PROVIDER_MODALITY_SUPPORT.keys())}\nAvailable modalities: text-only, image-only, audio-only, text-image, text-audio, image-audio, text-image-audio\n\nExamples:\n python capture_provider_stats.py # Run all providers/modalities\n python capture_provider_stats.py -p openai # Run all modalities for OpenAI\n python capture_provider_stats.py -p openai -m text-only # Run only text-only for OpenAI\n python capture_provider_stats.py -p anthropic -m text-image # Run only text-image for Anthropic\n \"\"\"\n )\n parser.add_argument(\n \"-p\", \"--provider\",\n nargs=\"+\",\n choices=list(PROVIDER_MODALITY_SUPPORT.keys()),\n help=\"Provider(s) to run. If not specified, runs all providers.\",\n )\n parser.add_argument(\n \"-m\", \"--modality\",\n nargs=\"+\",\n choices=[\"text-only\", \"image-only\", \"audio-only\", \"text-image\", \"text-audio\", \"image-audio\", \"text-image-audio\"],\n help=\"Modality(ies) to run. If not specified, runs all supported modalities for each provider.\",\n )\n args = parser.parse_args()\n\n messages_dir = os.path.join(\n os.path.dirname(__file__),\n \"..\",\n \"tests\",\n \"pytest\",\n \"data\",\n \"generator_messages\",\n )\n messages_dir = os.path.abspath(messages_dir)\n\n output_dir = os.path.join(\n os.path.dirname(__file__),\n \"provider_stats\",\n )\n output_dir = os.path.abspath(output_dir)\n\n print(f\"Loading messages from: {messages_dir}\")\n print(f\"Saving stats to: {output_dir}\\n\")\n\n # Determine which providers to run\n providers_to_run = args.provider if args.provider else list(PROVIDER_MODALITY_SUPPORT.keys())\n print(f\"Providers to run: {providers_to_run}\\n\")\n\n for provider in providers_to_run:\n config = PROVIDER_MODALITY_SUPPORT[provider]\n model = config[\"model\"]\n supported_modalities = config[\"supported_modalities\"]\n\n # Filter modalities if specified\n if args.modality:\n modalities_to_run = [m for m in args.modality if m in supported_modalities]\n if not modalities_to_run:\n print(f\"\\nProvider: {provider} - SKIPPED (none of {args.modality} supported)\")\n continue\n else:\n modalities_to_run = supported_modalities\n\n print(f\"\\nProvider: {provider} (model: {model})\")\n print(f\" Modalities to run: {modalities_to_run}\")\n\n for modality in modalities_to_run:\n print(f\"\\n Processing modality: {modality}\")\n\n try:\n messages = load_messages(modality, provider, messages_dir)\n print(f\" Loaded {len(messages)} messages from {modality}_{provider}.json\")\n\n stats = capture_stats_for_provider(provider, modality, messages, model)\n\n output_path = save_stats(stats, output_dir, provider, modality)\n print(f\" Saved to: {output_path}\")\n\n except FileNotFoundError as e:\n print(f\" SKIPPED: Message file not found - {e}\")\n except Exception as e:\n print(f\" ERROR: {e}\")\n import traceback\n traceback.print_exc()\n\n print(\"\\nDone!\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "scripts/generate_test_messages.py", "content": "#!/usr/bin/env python3\n\"\"\"\nScript to generate test messages for each provider/modality combination.\n\nThis script uses the Generator class directly to create message payloads.\nIt uses the 'generating_messages_only' flag to retrieve the exact messages\nthat would be sent to the provider without making an actual API call.\n\nSupported provider/modality combinations:\n- Anthropic: text-only, image-only, text-image (no audio support)\n- OpenAI: text-only, image-only, text-image\n- OpenAI-Audio: audio-only, text-audio\n- Gemini: all 7 modality combinations\n- Vertex AI: all 7 modality combinations\n- Azure: text-only, image-only, text-image\n\nOutput files are saved to: tests/pytest/data/generator_messages/\nFormat: {modality}_{provider}.json (e.g., text-only_anthropic.json)\n\"\"\"\n\nimport json\nimport os\nimport sys\n\nfrom pydantic import BaseModel, Field\n\n# Add project root to path\nsys.path.insert(0, os.path.join(os.path.dirname(__file__), \"..\"))\n\nfrom palimpzest.constants import Model, PromptStrategy\nfrom palimpzest.core.elements.records import DataRecord\nfrom palimpzest.core.lib.schemas import AudioFilepath, ImageFilepath, union_schemas\nfrom palimpzest.query.generators.generators import Generator\n\n\ndef generate_session_id(provider: str, modality: str) -> str:\n \"\"\"\n Generate a unique 12-character session ID for a provider/modality combination.\n This ensures each modality test has a unique prompt prefix, preventing cross-modality cache hits.\n The ID is deterministic based on provider+modality so regenerating produces consistent results.\n \"\"\"\n import hashlib\n hash_input = f\"{provider}_{modality}\"\n hash_hex = hashlib.md5(hash_input.encode()).hexdigest()\n return hash_hex[:12].upper()\n\nSTATIC_CONTEXT = \"\"\"\nWILDLIFE CONSERVATION & RESEARCH CENTER: SPECIES IDENTIFICATION MANUAL (v2025.1)\n\nSECTION 1: INTRODUCTION AND MISSION\nThe Wildlife Conservation & Research Center (WCRC) is dedicated to the preservation, study, and rehabilitation of diverse wildlife species.\nAll staff members, researchers, and volunteers must adhere to these protocols for accurate species identification and data collection.\nOur mission combines advanced biological sciences with conservation efforts to protect endangered and threatened populations worldwide.\n\nSECTION 2: MAMMAL IDENTIFICATION PROTOCOLS\n\n2.1 ELEPHANTS (Family Elephantidae):\n - African Savanna Elephant: Larger ears (shaped like Africa), concave back, two fingers on trunk tip. Weight: 5,000-14,000 lbs.\n - African Forest Elephant: Smaller stature, oval-shaped ears, straighter tusks pointing downward.\n - Asian Elephant: Smaller ears, convex back, one finger on trunk tip, twin domes on head. Weight: 4,000-11,000 lbs.\n - Vocalizations: Trumpeting (alarm/excitement), rumbling (long-distance communication), roaring (distress).\n\n2.2 BIG CATS (Family Felidae):\n - Lion (Panthera leo): Tawny coat, males have distinctive mane. Social, live in prides. Height: 3.5-4 ft at shoulder.\n - Tiger (Panthera tigris): Orange coat with black stripes, white underbelly. Solitary hunters. Largest cat species.\n - Leopard (Panthera pardus): Golden-yellow coat with rosette patterns. Excellent climbers, often cache prey in trees.\n - Cheetah (Acinonyx jubatus): Spotted coat, black \"tear marks\" from eyes to mouth. Fastest land animal (70 mph).\n - Vocalizations: Roaring (lions, tigers, leopards), chirping/purring (cheetahs cannot roar).\n\n2.3 BEARS (Family Ursidae):\n - Brown Bear (Ursus arctos): Large shoulder hump, dish-shaped face, long claws. Includes grizzly subspecies.\n - Black Bear (Ursus americanus): Straight facial profile, no shoulder hump, shorter claws. Most common North American bear.\n - Polar Bear (Ursus maritimus): White fur, longer neck, smaller ears. Marine mammal adapted to Arctic conditions.\n - Giant Panda (Ailuropoda melanoleuca): Black and white coloring, feeds almost exclusively on bamboo.\n - Vocalizations: Roaring, growling, huffing, jaw-popping (threat displays).\n\n2.4 PRIMATES (Order Primates):\n - Gorilla: Largest primate, silver-back males, knuckle-walking locomotion. Vocalizations include chest-beating, hooting.\n - Chimpanzee: Highly intelligent, uses tools, complex social structures. Vocalizations: pant-hoots, screams.\n - Orangutan: Red-orange fur, arboreal lifestyle, solitary. Long calls can travel over 1 km.\n - Gibbon: Smaller apes, brachiation locomotion, distinctive whooping songs for territorial marking.\n\nSECTION 3: BIRD IDENTIFICATION PROTOCOLS\n\n3.1 RAPTORS (Order Accipitriformes/Falconiformes):\n - Bald Eagle: White head and tail, yellow beak. Wingspan: 6-7.5 ft. Call: high-pitched chattering.\n - Golden Eagle: Dark brown plumage, golden nape. Powerful hunters of small mammals.\n - Peregrine Falcon: Blue-gray back, barred underparts. Fastest bird in dive (240+ mph).\n - Red-tailed Hawk: Brown back, pale underparts, distinctive red tail. Most common North American hawk.\n\n3.2 PARROTS (Order Psittaciformes):\n - Macaw: Large, colorful, long tail feathers. Powerful curved beaks. Highly social and vocal.\n - African Grey: Gray plumage, red tail. Exceptional mimicry and cognitive abilities.\n - Cockatoo: White or pink plumage, distinctive crest. Loud screeching vocalizations.\n\nSECTION 4: REPTILE IDENTIFICATION PROTOCOLS\n\n4.1 CROCODILIANS (Order Crocodilia):\n - American Alligator: Broad, U-shaped snout, dark coloration. Freshwater habitats.\n - Nile Crocodile: V-shaped snout, aggressive. Can reach 16-18 ft in length.\n - Gharial: Extremely narrow snout, fish-eating specialist. Critically endangered.\n\n4.2 LARGE SNAKES (Families Pythonidae/Boidae):\n - Reticulated Python: Longest snake species (up to 23 ft), complex geometric patterns.\n - Green Anaconda: Heaviest snake species, olive-green with black spots. Semi-aquatic.\n - King Cobra: Longest venomous snake (up to 18 ft), distinctive hood when threatened.\n\nSECTION 5: DATA COLLECTION AND ANALYSIS\n\n5.1 Visual Identification:\n - Document body shape, size, coloration, and distinctive markings.\n - Note behavioral characteristics and habitat context.\n - Use standardized photography protocols for pattern matching.\n\n5.2 Audio Identification:\n - Record vocalizations with frequency analysis equipment.\n - Tag recordings with behavioral context (territorial, mating, alarm, social).\n - Cross-reference with vocalization databases for species confirmation.\n\n5.3 Biometric Data:\n - Record body measurements according to species-specific protocols.\n - Document age indicators (teeth wear, plumage, etc.).\n - Collect genetic samples when possible for lineage verification.\n\nYou are an AI Research Assistant for the WCRC. Your job is to analyze data inputs (text descriptions, images, and/or audio recordings) and identify the species based on the characteristics described in this manual.\nAnalyze all provided inputs and determine the most likely species identification.\n\"\"\"\n\nclass TextInputSchema(BaseModel):\n \"\"\"Schema for text-only input.\"\"\"\n text: str = Field(description=\"Description of an animal\")\n age: int = Field(description=\"The age of the animal in years\")\n\n\nclass ImageInputSchema(BaseModel):\n \"\"\"Schema for image-only input.\"\"\"\n image_file: ImageFilepath = Field(description=\"File path to an image of an animal\")\n height: float = Field(description=\"The estimated height of the animal in cm\")\n\n\nclass AudioInputSchema(BaseModel):\n \"\"\"Schema for audio-only input.\"\"\"\n audio_file: AudioFilepath = Field(description=\"File path to an audio recording of an animal\")\n year: float = Field(description=\"The year the recording was made\")\n\n\n# Union schemas for multi-modal inputs\nTextImageInputSchema = union_schemas([TextInputSchema, ImageInputSchema])\nTextAudioInputSchema = union_schemas([TextInputSchema, AudioInputSchema])\nImageAudioInputSchema = union_schemas([ImageInputSchema, AudioInputSchema])\nTextImageAudioInputSchema = union_schemas([TextInputSchema, ImageInputSchema, AudioInputSchema])\n\n\nclass OutputSchema(BaseModel):\n \"\"\"Output schema for animal identification.\"\"\"\n animal: str = Field(description=\"The animal in the input\")\n\nMODALITY_CONFIGS = {\n \"text-only\": {\n \"input_schema\": TextInputSchema,\n \"data_item\": {\n \"text\": \"An elephant is a large gray animal with a trunk and big ears. It makes a trumpeting sound.\",\n \"age\": 15,\n },\n },\n \"image-only\": {\n \"input_schema\": ImageInputSchema,\n \"data_item\": {\n \"image_file\": \"tests/pytest/data/elephant.png\",\n \"height\": 304.5,\n },\n },\n \"audio-only\": {\n \"input_schema\": AudioInputSchema,\n \"data_item\": {\n \"audio_file\": \"tests/pytest/data/elephant.wav\",\n \"year\": 2020,\n },\n },\n \"text-image\": {\n \"input_schema\": TextImageInputSchema,\n \"data_item\": {\n \"text\": \"An elephant is a large gray animal with a trunk and big ears. It makes a trumpeting sound.\",\n \"age\": 15,\n \"image_file\": \"tests/pytest/data/elephant.png\",\n \"height\": 304.5,\n },\n },\n \"text-audio\": {\n \"input_schema\": TextAudioInputSchema,\n \"data_item\": {\n \"text\": \"An elephant is a large gray animal with a trunk and big ears. It makes a trumpeting sound.\",\n \"age\": 15,\n \"audio_file\": \"tests/pytest/data/elephant.wav\",\n \"year\": 2020,\n },\n },\n \"image-audio\": {\n \"input_schema\": ImageAudioInputSchema,\n \"data_item\": {\n \"image_file\": \"tests/pytest/data/elephant.png\",\n \"height\": 304.5,\n \"audio_file\": \"tests/pytest/data/elephant.wav\",\n \"year\": 2020,\n },\n },\n \"text-image-audio\": {\n \"input_schema\": TextImageAudioInputSchema,\n \"data_item\": {\n \"text\": \"An elephant is a large gray animal with a trunk and big ears. It makes a trumpeting sound.\",\n \"age\": 15,\n \"image_file\": \"tests/pytest/data/elephant.png\",\n \"height\": 304.5,\n \"audio_file\": \"tests/pytest/data/elephant.wav\",\n \"year\": 2020,\n },\n },\n}\n\n# Maps provider name to (Model enum, supported modalities)\nPROVIDER_CONFIGS = {\n \"anthropic\": {\n \"model\": Model.CLAUDE_4_5_SONNET,\n \"supported_modalities\": [\"text-only\", \"image-only\", \"text-image\"],\n },\n \"openai\": {\n \"model\": Model.GPT_4o,\n \"supported_modalities\": [\"text-only\", \"image-only\", \"text-image\"],\n },\n \"openai-audio\": {\n \"model\": Model.GPT_4o_AUDIO_PREVIEW,\n \"supported_modalities\": [\"audio-only\", \"text-audio\"],\n },\n \"gemini\": {\n \"model\": Model.GOOGLE_GEMINI_2_5_FLASH,\n \"supported_modalities\": [\n \"text-only\", \"image-only\", \"audio-only\",\n \"text-image\", \"text-audio\", \"image-audio\", \"text-image-audio\",\n ],\n },\n \"vertex_ai\": {\n \"model\": Model.GEMINI_2_5_FLASH,\n \"supported_modalities\": [\n \"text-only\", \"image-only\", \"audio-only\",\n \"text-image\", \"text-audio\", \"image-audio\", \"text-image-audio\",\n ],\n },\n \"azure\": {\n \"model\": Model.AZURE_GPT_4o,\n \"supported_modalities\": [\"text-only\", \"image-only\", \"text-image\"],\n },\n}\n\n\ndef save_messages(modality: str, provider: str, messages: list[dict], output_dir: str) -> str:\n \"\"\"\n Save messages to a JSON file.\n\n Args:\n modality: Modality name\n provider: Provider name\n messages: List of message dicts\n output_dir: Directory to save files\n\n Returns:\n Path to the saved file\n \"\"\"\n os.makedirs(output_dir, exist_ok=True)\n output_path = os.path.join(output_dir, f\"{modality}_{provider}.json\")\n\n # Convert messages to JSON-serializable format\n serializable_messages = []\n for msg in messages:\n serializable_msg = msg.copy()\n serializable_messages.append(serializable_msg)\n\n with open(output_path, \"w\") as f:\n json.dump(serializable_messages, f, indent=2, default=str)\n\n return output_path\n\n\ndef main():\n \"\"\"Generate and save messages for all provider/modality combinations.\"\"\"\n # Ensure the output directory follows the repository structure\n output_dir = os.path.join(\n os.path.dirname(__file__),\n \"..\",\n \"tests\",\n \"pytest\",\n \"data\",\n \"generator_messages\",\n )\n output_dir = os.path.abspath(output_dir)\n\n # Count total combinations\n total_combinations = sum(\n len(provider_config[\"supported_modalities\"])\n for provider_config in PROVIDER_CONFIGS.values()\n )\n\n print(f\"Generating test messages for {total_combinations} provider/modality combinations...\")\n print(f\"Output directory: {output_dir}\")\n print(f\"Static context length: ~{len(STATIC_CONTEXT.split())} words\\n\")\n\n generated_count = 0\n\n for provider, provider_config in PROVIDER_CONFIGS.items():\n model = provider_config[\"model\"]\n supported_modalities = provider_config[\"supported_modalities\"]\n\n print(f\"Provider: {provider} (model: {model.value})\")\n print(f\" Supported modalities: {supported_modalities}\")\n\n for modality in supported_modalities:\n config = MODALITY_CONFIGS[modality]\n print(f\" Generating: {modality}_{provider}\")\n\n try:\n # Prepare input record\n input_schema = config[\"input_schema\"]\n data_item = config[\"data_item\"]\n input_record = DataRecord(input_schema(**data_item), source_indices=[0])\n\n # Instantiate Generator\n generator = Generator(\n model=model,\n prompt_strategy=PromptStrategy.MAP,\n reasoning_effort=None,\n desc=STATIC_CONTEXT,\n )\n\n # Generate unique session ID for this provider/modality to prevent cross-modality cache hits\n session_id = generate_session_id(provider, modality)\n\n # Call the generator with the new flag\n # Pass cache_isolation_id to inject session ID at start of system/user prompts\n messages = generator(\n candidate=input_record,\n fields=OutputSchema.model_fields,\n output_schema=OutputSchema,\n generating_messages_only=True,\n cache_isolation_id=session_id,\n )\n\n # Manually save the messages using local helper\n output_path = save_messages(modality, provider, messages, output_dir)\n\n print(f\" Session ID: {session_id}\")\n print(f\" Saved to: {output_path}\")\n print(f\" Messages: {len(messages)}\")\n\n # Print message summary\n for i, msg in enumerate(messages):\n role = msg.get(\"role\", \"unknown\")\n msg_type = msg.get(\"type\", \"unknown\")\n content = msg.get(\"content\", \"\")\n content_len = len(content) if isinstance(content, str) else len(str(content))\n print(f\" [{i}] role={role}, type={msg_type}, len={content_len}\")\n\n generated_count += 1\n\n except Exception as e:\n print(f\" ERROR: {e}\")\n import traceback\n traceback.print_exc()\n\n print()\n\n print(f\"Done! Generated {generated_count}/{total_combinations} message files.\")\n\n\nif __name__ == \"__main__\":\n main()" }, { "path": "scripts/update_model_info.py", "content": "#!/usr/bin/env python3\n\"\"\"\nScript to automatically update pz_models_information.json with data from external sources.\n\nData Sources:\n- LiteLLM proxy /model/info endpoint: Dynamic model info (100% accuracy, prioritized)\n- LiteLLM model_prices_and_context_window.json: Cost and capability data (fallback)\n- MMLU-Pro leaderboard: Quality scores (fuzzy matching acceptable)\n- Artificial Analysis: Latency data (fuzzy matching acceptable)\n\nUsage:\n python scripts/update_model_info.py MODEL_ID [MODEL_ID ...] [--use-endpoint]\n\"\"\"\n\nimport argparse\nimport json\nimport os\nimport socket\nimport subprocess\nimport sys\nimport time\nfrom typing import Any\n\nimport requests\nimport yaml\n\n# Add src to path to import from palimpzest\nsys.path.insert(0, os.path.join(os.path.dirname(__file__), \"..\", \"src\"))\n\nfrom palimpzest.utils.model_info_helpers import (\n LATENCY_TPS_DATA,\n MMLU_PRO_SCORES,\n derive_model_flags,\n fuzzy_match_score,\n)\n\n# Constants\nLITELLM_URL = \"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json\"\nPZ_MODELS_PATH = os.path.join(\n os.path.dirname(__file__),\n \"..\",\n \"src\",\n \"palimpzest\",\n \"utils\",\n \"pz_models_information.json\",\n)\n\n# Provider mapping from LiteLLM prefixes to our provider strings\nPROVIDER_MAPPING = {\n \"openai\": \"openai\",\n \"anthropic\": \"anthropic\",\n \"claude\": \"anthropic\",\n \"vertex_ai\": \"vertex_ai\",\n \"gemini\": \"gemini\",\n \"together_ai\": \"together_ai\",\n \"together\": \"together_ai\",\n \"hosted_vllm\": \"hosted_vllm\",\n \"groq\": \"groq\",\n \"mistral\": \"mistral\",\n \"cohere\": \"cohere\",\n \"bedrock\": \"bedrock\",\n \"azure\": \"azure\",\n \"deepseek\": \"deepseek\",\n \"fireworks_ai\": \"fireworks_ai\",\n \"xai\": \"xai\",\n}\n\n# API key environment variable mapping\nAPI_KEY_MAPPING = {\n \"openai\": \"OPENAI_API_KEY\",\n \"azure\": \"AZURE_API_KEY\",\n \"anthropic\": \"ANTHROPIC_API_KEY\",\n \"vertex_ai\": \"GOOGLE_APPLICATION_CREDENTIALS\",\n \"gemini\": \"GEMINI_API_KEY\",\n \"together_ai\": \"TOGETHER_API_KEY\",\n \"hosted_vllm\": \"VLLM_API_KEY\",\n \"groq\": \"GROQ_API_KEY\",\n \"mistral\": \"MISTRAL_API_KEY\",\n \"cohere\": \"COHERE_API_KEY\",\n \"deepseek\": \"DEEPSEEK_API_KEY\",\n \"fireworks_ai\": \"FIREWORKS_API_KEY\",\n \"xai\": \"XAI_API_KEY\",\n}\n\n# Field mapping from LiteLLM endpoint to PZ format\nFIELD_MAPPING = [\n (\"usd_per_input_token\", \"input_cost_per_token\", None),\n (\"usd_per_output_token\", \"output_cost_per_token\", None),\n (\"usd_per_audio_input_token\", \"input_cost_per_audio_token\", None),\n (\"usd_per_audio_output_token\", \"output_cost_per_audio_token\", None),\n (\"usd_per_image_output_token\", \"output_cost_per_image_token\", None),\n (\"usd_per_cache_read_token\", \"cache_read_input_token_cost\", None),\n (\"usd_per_cache_creation_token\", \"cache_creation_input_token_cost\", None),\n (\"supports_prompt_caching\", \"supports_prompt_caching\", False),\n]\n\n# Boolean capability fields derived from endpoint\nCAPABILITY_MAPPING = [\n (\"is_vision_model\", \"supports_vision\", False),\n (\"is_audio_model\", \"supports_audio_input\", False),\n (\"is_reasoning_model\", \"supports_reasoning\", False),\n]\n\n# MMLU_PRO_SCORES, LATENCY_TPS_DATA, and fuzzy_match_score are imported from\n# palimpzest.utils.model_info_helpers\n\n# Alias for backwards compatibility in this script\nLATENCY_DATA = LATENCY_TPS_DATA\n\n\n# =============================================================================\n# LiteLLM Proxy Endpoint Functions\n# =============================================================================\n\ndef get_free_port() -> int:\n with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:\n s.bind((\"\", 0))\n return s.getsockname()[1]\n\n\ndef extract_provider(model_id: str) -> str:\n \"\"\"Extract provider from model ID.\"\"\"\n if \"/\" in model_id:\n prefix = model_id.split(\"/\")[0].lower()\n return PROVIDER_MAPPING.get(prefix, prefix)\n\n model_lower = model_id.lower()\n \n # OpenAI\n if any(x in model_lower for x in [\"gpt\", \"o1-\", \"o3-\", \"o4-\", \"dall-e\", \"whisper\"]):\n return \"openai\"\n \n # Anthropic\n if \"claude\" in model_lower:\n return \"anthropic\"\n \n # Google (Vertex AI / Gemini)\n if \"gemini\" in model_lower or \"bison\" in model_lower:\n return \"vertex_ai\"\n \n # Meta / Together / Llama\n if \"llama\" in model_lower:\n return \"together_ai\"\n \n # Mistral\n if \"mistral\" in model_lower or \"mixtral\" in model_lower:\n return \"mistral\"\n\n # DeepSeek\n if \"deepseek\" in model_lower:\n return \"deepseek\"\n\n return \"unknown\"\n\n\ndef get_api_key_env_var(provider: str) -> str | None:\n return API_KEY_MAPPING.get(provider)\n\n\ndef generate_config_yaml(model_ids: list[str]) -> str:\n config_id = 0\n config_filename = f\"litellm_config_{config_id}.yaml\"\n while not os.path.exists(config_filename):\n config_id += 1\n\n config_list = []\n for model_id in model_ids:\n provider = extract_provider(model_id)\n env_var_name = get_api_key_env_var(provider)\n api_key_val = f\"os.environ/{env_var_name}\" if env_var_name else None\n\n entry = {\n \"model_name\": model_id,\n \"litellm_params\": {\n \"model\": model_id,\n \"api_key\": api_key_val,\n },\n }\n config_list.append(entry)\n\n yaml_structure = {\"model_list\": config_list}\n with open(config_filename, \"w\") as f:\n yaml.dump(yaml_structure, f, default_flow_style=False, sort_keys=False)\n\n return config_filename\n\n\ndef fetch_dynamic_model_info(model_ids: list[str]) -> dict[str, Any]:\n if not model_ids:\n return {}\n\n port = get_free_port()\n proxy_url = f\"http://127.0.0.1:{port}\"\n config_filename = generate_config_yaml(model_ids)\n server_env = os.environ.copy()\n process = None\n dynamic_model_info = {}\n\n print(f\"Starting LiteLLM proxy on port {port} for {len(model_ids)} models...\")\n\n try:\n process = subprocess.Popen(\n [\"litellm\", \"--config\", config_filename, \"--port\", str(port)],\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n env=server_env,\n )\n\n server_ready = False\n max_retries = 30\n for i in range(max_retries):\n if process.poll() is not None:\n _, stderr = process.communicate()\n print(f\" LiteLLM process died unexpectedly: {stderr.decode()}\")\n break\n try:\n requests.get(f\"{proxy_url}/health/readiness\", timeout=1)\n server_ready = True\n print(f\" Server ready after {i + 1} attempts\")\n break\n except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):\n time.sleep(0.5)\n\n if not server_ready:\n print(\" Timeout: LiteLLM server failed to start within the limit.\")\n return {}\n\n try:\n response = requests.get(f\"{proxy_url}/model/info\", timeout=10)\n response.raise_for_status()\n model_data = response.json()\n\n if \"data\" in model_data and len(model_data[\"data\"]) > 0:\n for item in model_data[\"data\"]:\n model_name = item.get(\"model_name\")\n model_info = item.get(\"model_info\", {})\n dynamic_model_info[model_name] = model_info\n print(f\" Retrieved info for: {model_name}\")\n else:\n print(\" WARNING: No model data returned from endpoint\")\n except Exception as e:\n print(f\" Error fetching model info: {e}\")\n\n finally:\n if process:\n process.terminate()\n try:\n process.wait(timeout=5)\n except subprocess.TimeoutExpired:\n process.kill()\n\n if os.path.exists(config_filename):\n os.remove(config_filename)\n\n return dynamic_model_info\n\n\n# =============================================================================\n# Data Fetching Functions\n# =============================================================================\n\ndef fetch_litellm_data() -> dict[str, Any]:\n print(f\"Fetching LiteLLM data from {LITELLM_URL}...\")\n try:\n response = requests.get(LITELLM_URL, timeout=30)\n response.raise_for_status()\n data = response.json()\n print(f\" Found {len(data)} models in LiteLLM database\")\n return data\n except Exception as e:\n print(f\" Error fetching LiteLLM data: {e}\")\n return {}\n\n\ndef load_existing_data() -> dict[str, Any]:\n if os.path.exists(PZ_MODELS_PATH):\n with open(PZ_MODELS_PATH) as f:\n return json.load(f)\n return {}\n\n\ndef save_data(data: dict[str, Any]) -> None:\n with open(PZ_MODELS_PATH, \"w\") as f:\n json.dump(data, f, indent=4)\n print(f\" [System] Successfully saved to {PZ_MODELS_PATH}\")\n\n\n# =============================================================================\n# Matching and Conversion Functions\n# =============================================================================\n\n# fuzzy_match_score is imported from palimpzest.utils.model_info_helpers\n\n\ndef derive_model_flags_with_provider(model_id: str, provider: str) -> dict[str, bool]:\n \"\"\"Wrapper around derive_model_flags that also adds provider-specific flags.\"\"\"\n flags = derive_model_flags(model_id)\n if provider == \"hosted_vllm\":\n flags[\"is_vllm_model\"] = True\n return flags\n\n\n# =============================================================================\n# Interactive Review Functions\n# =============================================================================\n\ndef prompt_for_value(field_name: str, current_value: Any, value_type: str = \"any\") -> Any:\n while True:\n user_input = input(f\" > Enter new value for '{field_name}' (or press Enter to keep current): \").strip()\n if user_input == \"\":\n return current_value\n \n try:\n if user_input.lower() == \"none\":\n return None\n if value_type == \"float\":\n return float(user_input)\n elif value_type == \"int\":\n return int(user_input)\n elif value_type == \"bool\":\n return user_input.lower() in (\"true\", \"yes\", \"1\", \"y\")\n else:\n try:\n return json.loads(user_input)\n except json.JSONDecodeError:\n return user_input\n except ValueError as e:\n print(f\" Invalid input: {e}. Try again.\")\n\n\ndef review_field(\n field_name: str,\n value: Any,\n from_endpoint: bool,\n interactive: bool = True,\n value_type: str = \"any\"\n) -> tuple[Any, bool]:\n \"\"\"\n Review a single field.\n Logic:\n 1. If from_endpoint is True and value not None -> VERIFIED (return immediately)\n 2. If interactive -> Ask User (1. Correct, 2. Incorrect)\n \"\"\"\n if from_endpoint and value is not None:\n # Verified automatically by endpoint\n return value, False\n\n if not interactive:\n return value, False\n\n print(f\"\\n [Review] {field_name}: {value}\")\n if from_endpoint and value is None:\n print(\" (Source: Endpoint returned Null)\")\n else:\n print(\" (Source: Derived/Static/Fallback)\")\n\n while True:\n choice = input(\" 1. Yes, information is correct\\n 2. No, enter different value\\n Choice [1]: \").strip()\n if choice == \"\" or choice == \"1\":\n return value, False\n elif choice == \"2\":\n new_value = prompt_for_value(field_name, value, value_type)\n return new_value, True\n else:\n print(\" Invalid choice.\")\n\n\ndef convert_and_review_model(\n model_id: str,\n litellm_static: dict[str, Any] | None,\n litellm_dynamic: dict[str, Any] | None,\n existing_entry: dict[str, Any] | None,\n interactive: bool = True,\n) -> dict[str, Any]:\n \"\"\"\n 1. Aggregates all data into a Draft Entry.\n 2. Displays the Draft Entry (User can see Current State).\n 3. Iterates fields to Verify (Prioritizing endpoint).\n \"\"\"\n print(f\"\\n{'='*60}\")\n print(f\"PROCESSING: {model_id}\")\n print(f\"{'='*60}\")\n\n # --- PHASE 1: Build Draft Entry & Source Map ---\n \n endpoint_fields: set[str] = set()\n raw_data: dict[str, Any] = {}\n\n # 1. Base: Static Data\n if litellm_static:\n raw_data.update(litellm_static)\n \n # 2. Overlay: Dynamic Data (Priority)\n if litellm_dynamic:\n for key, val in litellm_dynamic.items():\n if val is not None:\n raw_data[key] = val\n endpoint_fields.add(key)\n\n # 3. Construct Candidate dictionary\n candidate = {}\n source_map = {} # Map field -> is_from_endpoint\n\n # Provider\n prov = raw_data.get(\"litellm_provider\") or extract_provider(model_id)\n candidate[\"provider\"] = prov\n source_map[\"provider\"] = \"litellm_provider\" in endpoint_fields\n\n # Costs & Caching\n for pz_field, litellm_field, default in FIELD_MAPPING:\n val = raw_data.get(litellm_field, default)\n candidate[pz_field] = val\n source_map[pz_field] = litellm_field in endpoint_fields\n\n # Capabilities\n for pz_field, litellm_field, default in CAPABILITY_MAPPING:\n val = raw_data.get(litellm_field, default)\n # Special logic for audio\n if pz_field == \"is_audio_model\":\n audio_in = raw_data.get(\"supports_audio_input\", False)\n audio_out = raw_data.get(\"supports_audio_output\", False)\n val = audio_in or audio_out\n source_map[pz_field] = (\"supports_audio_input\" in endpoint_fields or \n \"supports_audio_output\" in endpoint_fields)\n else:\n source_map[pz_field] = litellm_field in endpoint_fields\n candidate[pz_field] = val\n\n # Modes\n mode = raw_data.get(\"mode\", \"chat\")\n mode_src = \"mode\" in endpoint_fields\n candidate[\"is_text_model\"] = mode in [\"chat\", \"completion\"]\n source_map[\"is_text_model\"] = mode_src\n candidate[\"is_embedding_model\"] = mode == \"embedding\"\n source_map[\"is_embedding_model\"] = mode_src\n\n # Flags (Always derived, never endpoint)\n flags = derive_model_flags_with_provider(model_id, candidate[\"provider\"])\n for k, v in flags.items():\n candidate[k] = v\n source_map[k] = False\n\n # Scores / Latency (Fuzzy or Existing)\n mmlu = fuzzy_match_score(model_id, MMLU_PRO_SCORES)\n if mmlu is None and existing_entry:\n mmlu = existing_entry.get(\"MMLU_Pro_score\")\n candidate[\"MMLU_Pro_score\"] = mmlu\n source_map[\"MMLU_Pro_score\"] = False\n\n tps = fuzzy_match_score(model_id, LATENCY_DATA)\n sec_per_tok = round(1.0 / tps, 6) if tps else None\n if sec_per_tok is None and existing_entry:\n sec_per_tok = existing_entry.get(\"seconds_per_output_token\")\n candidate[\"seconds_per_output_token\"] = sec_per_tok\n source_map[\"seconds_per_output_token\"] = False\n \n # Audio Cache Read (check existing)\n acr = existing_entry.get(\"usd_per_audio_cache_read_token\") if existing_entry else None\n if acr is not None:\n candidate[\"usd_per_audio_cache_read_token\"] = acr\n source_map[\"usd_per_audio_cache_read_token\"] = False\n\n # Note\n if existing_entry and existing_entry.get(\"note\"):\n candidate[\"note\"] = existing_entry[\"note\"]\n source_map[\"note\"] = False\n\n # Sources\n src_list = [LITELLM_URL]\n if existing_entry and existing_entry.get(\"sources\"):\n existing_srcs = existing_entry[\"sources\"]\n if isinstance(existing_srcs, list):\n src_list = list(set(src_list + existing_srcs))\n elif existing_srcs:\n src_list = list(set(src_list + [existing_srcs]))\n candidate[\"sources\"] = src_list\n\n # --- PHASE 2: Display Current State ---\n \n print(\"\\n--- Current State (Draft) ---\")\n display_dict = {}\n for k, v in candidate.items():\n if k == \"sources\":\n continue\n src_label = \"ENDPOINT\" if source_map.get(k, False) and v is not None else \"DERIVED/STATIC\"\n display_dict[k] = f\"{v} [{src_label}]\"\n \n print(json.dumps(display_dict, indent=2))\n print(\"-\" * 30)\n\n # --- PHASE 3: Verification Loop ---\n\n final_entry = {}\n final_entry[\"sources\"] = candidate[\"sources\"]\n\n # Iterate over specific keys to ensure order and types\n \n # Provider\n final_entry[\"provider\"], _ = review_field(\n \"provider\", candidate[\"provider\"], source_map[\"provider\"], interactive, \"str\"\n )\n\n # All cost/cache fields\n for k in [f[0] for f in FIELD_MAPPING] + [\"usd_per_audio_cache_read_token\"]:\n if k in candidate:\n vtype = \"float\" if \"usd_\" in k else \"bool\"\n final_entry[k], _ = review_field(\n k, candidate[k], source_map.get(k, False), interactive, vtype\n )\n\n # Capabilities & Modes\n bool_keys = [f[0] for f in CAPABILITY_MAPPING] + [\"is_text_model\", \"is_embedding_model\"] + list(flags.keys())\n for k in bool_keys:\n if k in candidate:\n final_entry[k], _ = review_field(\n k, candidate[k], source_map.get(k, False), interactive, \"bool\"\n )\n\n # Stats\n final_entry[\"MMLU_Pro_score\"], _ = review_field(\n \"MMLU_Pro_score\", candidate[\"MMLU_Pro_score\"], False, interactive, \"float\"\n )\n final_entry[\"seconds_per_output_token\"], _ = review_field(\n \"seconds_per_output_token\", candidate[\"seconds_per_output_token\"], False, interactive, \"float\"\n )\n\n # Note\n if \"note\" in candidate:\n final_entry[\"note\"], _ = review_field(\n \"note\", candidate[\"note\"], False, interactive, \"str\"\n )\n\n # Cleanup Nulls\n cleaned_entry = {k: v for k, v in final_entry.items() if v is not None}\n \n return cleaned_entry\n\n\ndef update_model(\n model_id: str,\n existing_data: dict[str, Any],\n litellm_static: dict[str, Any],\n litellm_dynamic: dict[str, Any] | None = None,\n interactive: bool = True,\n) -> dict[str, Any] | None:\n static_entry = None\n if model_id in litellm_static:\n static_entry = litellm_static[model_id]\n else:\n if \"/\" in model_id:\n model_name = model_id.split(\"/\", 1)[1]\n if model_name in litellm_static:\n static_entry = litellm_static[model_name]\n\n dynamic_entry = litellm_dynamic.get(model_id) if litellm_dynamic else None\n \n if static_entry is None and dynamic_entry is None:\n print(f\"\\n WARNING: No LiteLLM data found for {model_id}\")\n \n existing_entry = existing_data.get(model_id)\n\n new_entry = convert_and_review_model(\n model_id,\n static_entry,\n dynamic_entry,\n existing_entry,\n interactive=interactive,\n )\n return new_entry\n\n\ndef process_models(\n model_ids: list[str],\n existing_data: dict[str, Any],\n litellm_static: dict[str, Any],\n use_endpoint: bool = False,\n interactive: bool = True,\n skip_existing: bool = False,\n) -> None:\n \"\"\"\n Process models and (if interactive is True) ask user whether to write each one to file.\n \"\"\"\n litellm_dynamic = None\n if use_endpoint:\n litellm_dynamic = fetch_dynamic_model_info(model_ids)\n\n # We work on the existing_data dictionary directly so we can save incrementally\n current_data_state = existing_data.copy()\n\n for model_id in model_ids:\n # Check if model exists and if we should skip it\n if skip_existing and model_id in current_data_state:\n print(f\"\\n [System] Model '{model_id}' already exists in file. Skipping.\")\n continue\n\n new_entry = update_model(\n model_id, current_data_state, litellm_static, litellm_dynamic,\n interactive=interactive\n )\n\n if new_entry:\n # Display Final Result\n print(\"\\n\" + \"-\"*30)\n print(f\"FINAL JSON FOR: {model_id}\")\n print(json.dumps(new_entry, indent=2))\n print(\"-\" * 30)\n\n # Ask user to write to file\n should_save = True\n if interactive:\n confirm = input(f\"Write '{model_id}' to json file? [y/N]: \").strip().lower()\n should_save = confirm == 'y'\n\n if should_save:\n current_data_state[model_id] = new_entry\n save_data(current_data_state)\n else:\n print(f\" [System] Skipped saving {model_id}.\")\n\n\n# =============================================================================\n# Main Entry Point\n# =============================================================================\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"Update pz_models_information.json with external data sources\",\n formatter_class=argparse.RawDescriptionHelpFormatter\n )\n parser.add_argument(\"model_ids\", nargs=\"*\", help=\"Model IDs to update\")\n parser.add_argument(\"--use-endpoint\", action=\"store_true\", help=\"Fetch dynamic info\")\n parser.add_argument(\"--non-interactive\", action=\"store_true\", help=\"Skip review and auto-save\")\n parser.add_argument(\"--update-all\", action=\"store_true\", help=\"Update all existing\")\n\n args = parser.parse_args()\n\n litellm_static = fetch_litellm_data()\n if not litellm_static:\n return\n\n existing_data = load_existing_data()\n\n skip_existing = False\n if args.update_all:\n model_ids = list(existing_data.keys())\n elif args.model_ids:\n model_ids = args.model_ids\n skip_existing = True\n else:\n parser.print_help()\n return\n\n interactive = not args.non_interactive\n\n # Run the main processing loop\n process_models(\n model_ids,\n existing_data,\n litellm_static,\n use_endpoint=args.use_endpoint,\n interactive=interactive,\n skip_existing=skip_existing,\n )\n\n print(\"\\nAll operations complete.\")\n\nif __name__ == \"__main__\":\n main()" }, { "path": "src/palimpzest/__init__.py", "content": "import logging\n\nfrom palimpzest.constants import Cardinality, Model\nfrom palimpzest.core.data.context import Context, TextFileContext\nfrom palimpzest.core.data.dataset import Dataset\nfrom palimpzest.core.data.iter_dataset import (\n AudioFileDataset,\n HTMLFileDataset,\n ImageFileDataset,\n IterDataset,\n MemoryDataset,\n PDFFileDataset,\n TextFileDataset,\n XLSFileDataset,\n)\nfrom palimpzest.core.elements.groupbysig import GroupBySig\nfrom palimpzest.core.lib.schemas import AudioBase64, AudioFilepath, ImageBase64, ImageFilepath, ImageURL\nfrom palimpzest.policy import (\n MaxQuality,\n MaxQualityAtFixedCost,\n MaxQualityAtFixedTime,\n MinCost,\n MinCostAtFixedQuality,\n MinTime,\n MinTimeAtFixedQuality,\n PlanCost,\n Policy,\n)\nfrom palimpzest.query.processor.config import QueryProcessorConfig\nfrom palimpzest.validator.validator import Validator\n\n# Initialize the root logger\nlogging.getLogger(__name__).addHandler(logging.NullHandler())\n\n__all__ = [\n # constants\n \"Cardinality\",\n \"Model\",\n # core\n \"GroupBySig\",\n \"Context\",\n \"TextFileContext\",\n \"Dataset\",\n \"IterDataset\",\n \"AudioFileDataset\",\n \"MemoryDataset\",\n \"HTMLFileDataset\",\n \"ImageFileDataset\",\n \"PDFFileDataset\",\n \"TextFileDataset\",\n \"XLSFileDataset\",\n # schemas\n \"AudioBase64\",\n \"AudioFilepath\",\n \"ImageBase64\",\n \"ImageFilepath\",\n \"ImageURL\",\n # policy\n \"MaxQuality\",\n \"MaxQualityAtFixedCost\",\n \"MaxQualityAtFixedTime\",\n \"MinCost\",\n \"MinCostAtFixedQuality\",\n \"MinTime\",\n \"MinTimeAtFixedQuality\",\n \"PlanCost\",\n \"Policy\",\n # query\n \"QueryProcessorConfig\",\n # validator\n \"Validator\",\n]\n" }, { "path": "src/palimpzest/agents/__init__.py", "content": "" }, { "path": "src/palimpzest/agents/compute_agents.py", "content": "" }, { "path": "src/palimpzest/agents/search_agents.py", "content": "import json\nimport textwrap\nimport time\nfrom collections.abc import Generator\nfrom typing import TYPE_CHECKING, Any\n\nfrom rich.console import Group\nfrom rich.live import Live\nfrom rich.markdown import Markdown\nfrom rich.rule import Rule\nfrom rich.text import Text\n\nif TYPE_CHECKING:\n import PIL.Image\n\nfrom smolagents.agent_types import handle_agent_output_types\nfrom smolagents.agents import (\n ActionOutput,\n CodeAgent,\n FinalAnswerPromptTemplate,\n ManagedAgentPromptTemplate,\n PlanningPromptTemplate,\n PromptTemplates,\n RunResult,\n ToolOutput,\n populate_template,\n)\nfrom smolagents.local_python_executor import fix_final_answer_code\nfrom smolagents.memory import (\n ActionStep,\n FinalAnswerStep,\n PlanningStep,\n SystemPromptStep,\n TaskStep,\n Timing,\n TokenUsage,\n ToolCall,\n)\nfrom smolagents.models import (\n CODEAGENT_RESPONSE_FORMAT,\n ChatMessage,\n ChatMessageStreamDelta,\n MessageRole,\n agglomerate_stream_deltas,\n)\nfrom smolagents.monitoring import YELLOW_HEX, LogLevel\nfrom smolagents.utils import (\n AgentError,\n AgentExecutionError,\n AgentGenerationError,\n AgentMaxStepsError,\n AgentParsingError,\n extract_code_from_text,\n parse_code_blobs,\n truncate_content,\n)\n\nfrom palimpzest.prompts import (\n CODE_AGENT_SYSTEM_PROMPT,\n DATA_DISCOVERY_AGENT_INITIAL_PLAN_PROMPT,\n DATA_DISCOVERY_AGENT_REPORT_PROMPT,\n DATA_DISCOVERY_AGENT_TASK_PROMPT,\n DATA_DISCOVERY_AGENT_UPDATE_PLAN_POST_MESSAGES_PROMPT,\n DATA_DISCOVERY_AGENT_UPDATE_PLAN_PRE_MESSAGES_PROMPT,\n FINAL_ANSWER_POST_MESSAGES_PROMPT,\n FINAL_ANSWER_PRE_MESSAGES_PROMPT,\n)\n\n\n# TODO: make this use memory the way you want\nclass PZBaseAgent(CodeAgent):\n def __init__(self, run_id: str, context_description: str, *args, **kwargs):\n # memory_config = {\n # \"vector_store\": {\n # \"provider\": \"chroma\",\n # \"config\": {\n # \"collection_name\": f\"palimpzest-memory-{self.__class__.__name__}\",\n # \"path\": \"./pz-chroma\",\n # }\n # }\n # }\n # self.pz_memory = Memory.from_config(memory_config)\n self.run_id = run_id\n self.context_description = context_description\n super().__init__(*args, **kwargs)\n\n def write_memory_to_messages(\n self,\n summary_mode: bool = False,\n ) -> list[ChatMessage]:\n \"\"\"\n Reads past llm_outputs, actions, and observations or errors from the memory into a series of messages\n that can be used as input to the LLM. Adds a number of keywords (such as PLAN, error, etc) to help\n the LLM.\n \"\"\"\n messages = self.memory.system_prompt.to_messages(summary_mode=summary_mode)\n for memory_step in self.memory.steps:\n messages.extend(memory_step.to_messages(summary_mode=summary_mode))\n return messages\n\n def _generate_planning_step(\n self, task, is_first_step: bool, step: int\n ) -> Generator[ChatMessageStreamDelta | PlanningStep]:\n start_time = time.time()\n if is_first_step:\n input_messages = [\n ChatMessage(\n role=MessageRole.USER,\n content=[\n {\n \"type\": \"text\",\n \"text\": populate_template(\n self.prompt_templates[\"planning\"][\"initial_plan\"],\n variables={\"task\": task, \"tools\": self.tools, \"managed_agents\": self.managed_agents, \"context_description\": self.context_description},\n ),\n }\n ],\n )\n ]\n if self.stream_outputs and hasattr(self.model, \"generate_stream\"):\n plan_message_content = \"\"\n output_stream = self.model.generate_stream(input_messages, stop_sequences=[\"\"]) # type: ignore\n input_tokens, output_tokens = 0, 0\n with Live(\"\", console=self.logger.console, vertical_overflow=\"visible\") as live:\n for event in output_stream:\n if event.content is not None:\n plan_message_content += event.content\n live.update(Markdown(plan_message_content))\n if event.token_usage:\n output_tokens += event.token_usage.output_tokens\n input_tokens = event.token_usage.input_tokens\n yield event\n else:\n plan_message = self.model.generate(input_messages, stop_sequences=[\"\"])\n plan_message_content = plan_message.content\n input_tokens, output_tokens = (\n (\n plan_message.token_usage.input_tokens,\n plan_message.token_usage.output_tokens,\n )\n if plan_message.token_usage\n else (None, None)\n )\n plan = textwrap.dedent(\n f\"\"\"Here are the facts I know and the plan of action that I will follow to solve the task:\\n```\\n{plan_message_content}\\n```\"\"\"\n )\n else:\n # Summary mode removes the system prompt and previous planning messages output by the model.\n # Removing previous planning messages avoids influencing too much the new plan.\n memory_messages = self.write_memory_to_messages(summary_mode=True)\n plan_update_pre = ChatMessage(\n role=MessageRole.SYSTEM,\n content=[\n {\n \"type\": \"text\",\n \"text\": populate_template(\n self.prompt_templates[\"planning\"][\"update_plan_pre_messages\"], variables={\"task\": task, \"context_description\": self.context_description}\n ),\n }\n ],\n )\n plan_update_post = ChatMessage(\n role=MessageRole.USER,\n content=[\n {\n \"type\": \"text\",\n \"text\": populate_template(\n self.prompt_templates[\"planning\"][\"update_plan_post_messages\"],\n variables={\n \"task\": task,\n \"tools\": self.tools,\n \"managed_agents\": self.managed_agents,\n \"remaining_steps\": (self.max_steps - step),\n \"context_description\": self.context_description,\n },\n ),\n }\n ],\n )\n input_messages = [plan_update_pre] + memory_messages + [plan_update_post]\n if self.stream_outputs and hasattr(self.model, \"generate_stream\"):\n plan_message_content = \"\"\n input_tokens, output_tokens = 0, 0\n with Live(\"\", console=self.logger.console, vertical_overflow=\"visible\") as live:\n for event in self.model.generate_stream(\n input_messages,\n stop_sequences=[\"\"],\n ): # type: ignore\n if event.content is not None:\n plan_message_content += event.content\n live.update(Markdown(plan_message_content))\n if event.token_usage:\n output_tokens += event.token_usage.output_tokens\n input_tokens = event.token_usage.input_tokens\n yield event\n else:\n plan_message = self.model.generate(input_messages, stop_sequences=[\"\"])\n plan_message_content = plan_message.content\n if plan_message.token_usage is not None:\n input_tokens, output_tokens = (\n plan_message.token_usage.input_tokens,\n plan_message.token_usage.output_tokens,\n )\n plan = textwrap.dedent(\n f\"\"\"I still need to solve the task I was given:\\n```\\n{self.task}\\n```\\n\\nHere are the facts I know and my new/updated plan of action to solve the task:\\n```\\n{plan_message_content}\\n```\"\"\"\n )\n log_headline = \"Initial plan\" if is_first_step else \"Updated plan\"\n self.logger.log(Rule(f\"[bold]{log_headline}\", style=\"orange\"), Text(plan), level=LogLevel.INFO)\n yield PlanningStep(\n model_input_messages=input_messages,\n plan=plan,\n model_output_message=ChatMessage(role=MessageRole.ASSISTANT, content=plan_message_content),\n token_usage=TokenUsage(input_tokens=input_tokens, output_tokens=output_tokens),\n timing=Timing(start_time=start_time, end_time=time.time()),\n )\n\n # def _curate_messages(self, input_messages: list[ChatMessage]) -> list[ChatMessage]:\n # \"\"\"\n # Try returning:\n # - System Prompt + task\n # - Current Plan\n # - Summary of previous conversation\n # \"\"\"\n # # initialize with the system prompt & original task\n # curated_messages = input_messages[:2]\n\n # # find the last planning step message\n # idx = len(self.memory.steps) - 1\n # while idx > -1:\n # step = self.memory.steps[idx]\n # if isinstance(step, PlanningStep):\n # curated_messages.append(step.model_output_message)\n # break\n # idx -= 1\n\n # # add summary of chat history\n # history = self.pz_memory.search(\"A condensed summary of the execution trace of the agent.\", run_id=self.run_id)\n # for msg in history[\"results\"]:\n # pass\n\n # return curated_messages\n\n def _step_stream(self, memory_step: ActionStep) -> Generator[ChatMessageStreamDelta | ActionOutput | ToolOutput]:\n \"\"\"\n Perform one step in the ReAct framework: the agent thinks, acts, and observes the result.\n Yields ChatMessageStreamDelta during the run if streaming is enabled.\n At the end, yields either None if the step is not final, or the final answer.\n \"\"\"\n memory_messages = self.write_memory_to_messages()\n\n input_messages = memory_messages.copy()\n\n ### Generate model output ###\n memory_step.model_input_messages = input_messages\n try:\n additional_args: dict[str, Any] = {}\n if self.grammar:\n additional_args[\"grammar\"] = self.grammar\n if self._use_structured_outputs_internally:\n additional_args[\"response_format\"] = CODEAGENT_RESPONSE_FORMAT\n if self.stream_outputs:\n output_stream = self.model.generate_stream(\n input_messages,\n stop_sequences=[\"\", \"Observation:\", \"Calling tools:\"],\n **additional_args,\n )\n chat_message_stream_deltas: list[ChatMessageStreamDelta] = []\n with Live(\"\", console=self.logger.console, vertical_overflow=\"visible\") as live:\n for event in output_stream:\n chat_message_stream_deltas.append(event)\n live.update(\n Markdown(agglomerate_stream_deltas(chat_message_stream_deltas).render_as_markdown())\n )\n yield event\n chat_message = agglomerate_stream_deltas(chat_message_stream_deltas)\n memory_step.model_output_message = chat_message\n output_text = chat_message.content\n else:\n chat_message: ChatMessage = self.model.generate(\n input_messages,\n stop_sequences=[\"\", \"Observation:\", \"Calling tools:\"],\n **additional_args,\n )\n memory_step.model_output_message = chat_message\n output_text = chat_message.content\n self.logger.log_markdown(\n content=output_text,\n title=\"Output message of the LLM:\",\n level=LogLevel.DEBUG,\n )\n\n # This adds sequence to the history.\n # This will nudge ulterior LLM calls to finish with , thus efficiently stopping generation.\n if output_text and output_text.strip().endswith(\"```\"):\n output_text += \"\"\n memory_step.model_output_message.content = output_text\n\n memory_step.token_usage = chat_message.token_usage\n memory_step.model_output = output_text\n except Exception as e:\n raise AgentGenerationError(f\"Error in generating model output:\\n{e}\", self.logger) from e\n\n ### Parse output ###\n try:\n if self._use_structured_outputs_internally:\n code_action = json.loads(output_text)[\"code\"]\n code_action = extract_code_from_text(code_action) or code_action\n else:\n code_action = parse_code_blobs(output_text)\n code_action = fix_final_answer_code(code_action)\n memory_step.code_action = code_action\n except Exception as e:\n error_msg = f\"Error in code parsing:\\n{e}\\nMake sure to provide correct code blobs.\"\n raise AgentParsingError(error_msg, self.logger) from e\n\n memory_step.tool_calls = [\n ToolCall(\n name=\"python_interpreter\",\n arguments=code_action,\n id=f\"call_{len(self.memory.steps)}\",\n )\n ]\n\n ### Execute action ###\n self.logger.log_code(title=\"Executing parsed code:\", content=code_action, level=LogLevel.INFO)\n is_final_answer = False\n try:\n output, execution_logs, is_final_answer = self.python_executor(code_action)\n execution_outputs_console = []\n if len(execution_logs) > 0:\n execution_outputs_console += [\n Text(\"Execution logs:\", style=\"bold\"),\n Text(execution_logs),\n ]\n observation = \"Execution logs:\\n\" + execution_logs\n except Exception as e:\n if hasattr(self.python_executor, \"state\") and \"_print_outputs\" in self.python_executor.state:\n execution_logs = str(self.python_executor.state[\"_print_outputs\"])\n if len(execution_logs) > 0:\n execution_outputs_console = [\n Text(\"Execution logs:\", style=\"bold\"),\n Text(execution_logs),\n ]\n memory_step.observations = \"Execution logs:\\n\" + execution_logs\n self.logger.log(Group(*execution_outputs_console), level=LogLevel.INFO)\n error_msg = str(e)\n if \"Import of \" in error_msg and \" is not allowed\" in error_msg:\n self.logger.log(\n \"[bold red]Warning to user: Code execution failed due to an unauthorized import - Consider passing said import under `additional_authorized_imports` when initializing your CodeAgent.\",\n level=LogLevel.INFO,\n )\n raise AgentExecutionError(error_msg, self.logger) from e\n\n truncated_output = truncate_content(str(output))\n observation += \"Last output from code snippet:\\n\" + truncated_output\n memory_step.observations = observation\n \n # # TODO: add output to self.pz_memory\n # def get_role(msg_role):\n # return str(msg_role).split(\".\")[-1].lower()\n\n # messages = [\n # {\"role\": get_role(memory_step.model_output_message.role), \"content\": memory_step.model_output_message.content},\n # {\"role\": \"user\", \"content\": memory_step.observations},\n # ]\n # self.pz_memory.add(messages, run_id=self.run_id, agent_id=self.name)\n\n execution_outputs_console += [\n Text(\n f\"{('Out - Final answer' if is_final_answer else 'Out')}: {truncated_output}\",\n style=(f\"bold {YELLOW_HEX}\" if is_final_answer else \"\"),\n ),\n ]\n self.logger.log(Group(*execution_outputs_console), level=LogLevel.INFO)\n memory_step.action_output = output\n yield ActionOutput(output=output, is_final_answer=is_final_answer)\n\n def _run_stream(\n self, task: str, max_steps: int, images: list[\"PIL.Image.Image\"] | None = None\n ) -> Generator[ActionStep | PlanningStep | FinalAnswerStep | ChatMessageStreamDelta]:\n \"\"\"\n Execute the agent.\n \"\"\"\n self.step_number = 1\n returned_final_answer = False\n while not returned_final_answer and self.step_number <= self.max_steps:\n # Run a planning step if scheduled\n if self.planning_interval is not None and (\n self.step_number == 1 or (self.step_number - 1) % self.planning_interval == 0\n ):\n planning_start_time = time.time()\n planning_step = None\n for element in self._generate_planning_step(\n self.task, is_first_step=len(self.memory.steps) == 1, step=self.step_number\n ): # Don't use the attribute step_number here, because there can be steps from previous runs\n yield element\n planning_step = element\n assert isinstance(planning_step, PlanningStep) # Last yielded element should be a PlanningStep\n self.memory.steps.append(planning_step)\n planning_end_time = time.time()\n planning_step.timing = Timing(\n start_time=planning_start_time,\n end_time=planning_end_time,\n )\n\n # Start action step!\n action_step_start_time = time.time()\n action_step = ActionStep(\n step_number=self.step_number,\n timing=Timing(start_time=action_step_start_time),\n observations_images=images,\n )\n self.logger.log_rule(f\"Step {self.step_number}\", level=LogLevel.INFO)\n try:\n for output in self._step_stream(action_step):\n # Yield streaming deltas\n if not isinstance(output, (ActionOutput, ToolOutput)):\n # non-action, non-tool output\n yield output\n\n if isinstance(output, (ActionOutput, ToolOutput)) and output.is_final_answer:\n if self.final_answer_checks:\n self._validate_final_answer(output.output)\n returned_final_answer = True\n action_step.is_final_answer = True\n final_answer = output.output\n # handle final step\n except AgentGenerationError as e:\n # Agent generation errors are not caused by a Model error but an implementation error: so we should raise them and exit.\n raise e\n except AgentError as e:\n # Other AgentError types are caused by the Model, so we should log them and iterate.\n action_step.error = e\n finally:\n self._finalize_step(action_step)\n self.memory.steps.append(action_step)\n yield action_step\n self.step_number += 1\n\n if not returned_final_answer and self.step_number == self.max_steps + 1:\n final_answer = self._handle_max_steps_reached(self.task, images)\n yield action_step\n yield FinalAnswerStep(handle_agent_output_types(final_answer))\n\n def run(\n self,\n task: str,\n stream: bool = False,\n reset: bool = True,\n images: list[\"PIL.Image.Image\"] | None = None,\n additional_args: dict | None = None,\n max_steps: int | None = None,\n ):\n \"\"\"\n Run the agent for the given task.\n\n Args:\n task (`str`): Task to perform.\n stream (`bool`): Whether to run in streaming mode.\n If `True`, returns a generator that yields each step as it is executed. You must iterate over this generator to process the individual steps (e.g., using a for loop or `next()`).\n If `False`, executes all steps internally and returns only the final answer after completion.\n reset (`bool`): Whether to reset the conversation or keep it going from previous run.\n images (`list[PIL.Image.Image]`, *optional*): Image(s) objects.\n additional_args (`dict`, *optional*): Any other variables that you want to pass to the agent run, for instance images or dataframes. Give them clear names!\n max_steps (`int`, *optional*): Maximum number of steps the agent can take to solve the task. if not provided, will use the agent's default value.\n\n Example:\n ```py\n from smolagents import CodeAgent\n agent = CodeAgent(tools=[])\n agent.run(\"What is the result of 2 power 3.7384?\")\n ```\n \"\"\"\n max_steps = max_steps or self.max_steps\n self.task = task\n self.interrupt_switch = False\n if additional_args is not None:\n self.state.update(additional_args)\n self.task += f\"\"\"\nYou have been provided with these additional arguments, that you can access using the keys as variables in your python code:\n{str(additional_args)}.\"\"\"\n\n self.memory.system_prompt = SystemPromptStep(system_prompt=self.system_prompt)\n if reset:\n self.memory.reset()\n self.monitor.reset()\n\n self.logger.log_task(\n content=self.task.strip(),\n subtitle=f\"{type(self.model).__name__} - {(self.model.model_id if hasattr(self.model, 'model_id') else '')}\",\n level=LogLevel.INFO,\n title=self.name if hasattr(self, \"name\") else None,\n )\n self.memory.steps.append(TaskStep(task=self.task, task_images=images))\n\n if getattr(self, \"python_executor\", None):\n self.python_executor.send_variables(variables=self.state)\n self.python_executor.send_tools({**self.tools, **self.managed_agents})\n\n if stream:\n # The steps are returned as they are executed through a generator to iterate on.\n return self._run_stream(task=self.task, max_steps=max_steps, images=images)\n run_start_time = time.time()\n # Outputs are returned only at the end. We only look at the last step.\n\n steps = list(self._run_stream(task=self.task, max_steps=max_steps, images=images))\n assert isinstance(steps[-1], FinalAnswerStep)\n output = steps[-1].output\n\n if self.return_full_result:\n total_input_tokens = 0\n total_output_tokens = 0\n correct_token_usage = True\n for step in self.memory.steps:\n if isinstance(step, (ActionStep, PlanningStep)):\n if step.token_usage is None:\n correct_token_usage = False\n break\n else:\n total_input_tokens += step.token_usage.input_tokens\n total_output_tokens += step.token_usage.output_tokens\n if correct_token_usage:\n token_usage = TokenUsage(input_tokens=total_input_tokens, output_tokens=total_output_tokens)\n else:\n token_usage = None\n\n if self.memory.steps and isinstance(getattr(self.memory.steps[-1], \"error\", None), AgentMaxStepsError):\n state = \"max_steps_error\"\n else:\n state = \"success\"\n\n messages = self.memory.get_full_steps()\n\n return RunResult(\n output=output,\n token_usage=token_usage,\n messages=messages,\n timing=Timing(start_time=run_start_time, end_time=time.time()),\n state=state,\n )\n\n return output\n\n\nclass PZBaseManagedAgent(PZBaseAgent):\n\n def __call__(self, task: str, **kwargs):\n \"\"\"Adds additional prompting for the managed agent, runs it, and wraps the output.\n This method is called only by a managed agent.\n \"\"\"\n full_task = populate_template(\n self.prompt_templates[\"managed_agent\"][\"task\"],\n variables=dict(name=self.name, task=task, context_description=self.context_description),\n )\n result = self.run(full_task, **kwargs)\n report = result.output if isinstance(result, RunResult) else result\n answer = populate_template(\n self.prompt_templates[\"managed_agent\"][\"report\"], variables=dict(name=self.name, final_answer=report)\n )\n if self.provide_run_summary:\n answer += \"\\n\\nFor more detail, find below a summary of this agent's work:\\n\\n\"\n for message in self.write_memory_to_messages(summary_mode=True):\n content = message.content\n answer += \"\\n\" + truncate_content(str(content)) + \"\\n---\"\n answer += \"\\n\"\n return answer\n\n\nclass DataDiscoveryAgent(PZBaseManagedAgent):\n def __init__(self, run_id: str, context_description: str, *args, **kwargs):\n self.description = \"\"\"A team member that will search a data repository to find files which help to answer your question.\n Ask him for all your questions that require searching a repository of relevant data.\n Provide him as much context as possible, in particular if you need to search on a specific timeframe!\n And don't hesitate to provide him with a complex search task, like finding a difference between two files.\n Your request must be a real sentence, not a keyword search! Like \"Find me this information (...)\" rather than a few keywords.\n \"\"\"\n prompt_templates = PromptTemplates(\n system_prompt=CODE_AGENT_SYSTEM_PROMPT,\n planning=PlanningPromptTemplate(\n initial_plan=DATA_DISCOVERY_AGENT_INITIAL_PLAN_PROMPT,\n update_plan_pre_messages=DATA_DISCOVERY_AGENT_UPDATE_PLAN_PRE_MESSAGES_PROMPT,\n update_plan_post_messages=DATA_DISCOVERY_AGENT_UPDATE_PLAN_POST_MESSAGES_PROMPT,\n ),\n managed_agent=ManagedAgentPromptTemplate(task=DATA_DISCOVERY_AGENT_TASK_PROMPT, report=DATA_DISCOVERY_AGENT_REPORT_PROMPT),\n final_answer=FinalAnswerPromptTemplate(pre_messages=FINAL_ANSWER_PRE_MESSAGES_PROMPT, post_messages=FINAL_ANSWER_POST_MESSAGES_PROMPT),\n )\n\n super().__init__(\n *args,\n run_id=run_id,\n context_description=context_description,\n prompt_templates=prompt_templates,\n max_steps=20,\n verbosity_level=2,\n planning_interval=4,\n name=\"data_discovery_agent\",\n description=self.description,\n provide_run_summary=True,\n **kwargs,\n )\n self.prompt_templates[\"managed_agent\"][\"task\"] += \"\"\"Additionally, if after some searching you find out that you need more information to answer the question, you can use `final_answer` with your request for clarification as argument to request for more information.\"\"\"\n\n\nclass SearchManagerAgent(PZBaseAgent):\n def __init__(self, run_id: str, context_description: str, *args, **kwargs):\n prompt_templates = PromptTemplates(\n system_prompt=CODE_AGENT_SYSTEM_PROMPT,\n planning=PlanningPromptTemplate(\n initial_plan=DATA_DISCOVERY_AGENT_INITIAL_PLAN_PROMPT,\n update_plan_pre_messages=DATA_DISCOVERY_AGENT_UPDATE_PLAN_PRE_MESSAGES_PROMPT,\n update_plan_post_messages=DATA_DISCOVERY_AGENT_UPDATE_PLAN_POST_MESSAGES_PROMPT,\n ),\n managed_agent=ManagedAgentPromptTemplate(task=DATA_DISCOVERY_AGENT_TASK_PROMPT, report=DATA_DISCOVERY_AGENT_REPORT_PROMPT),\n final_answer=FinalAnswerPromptTemplate(pre_messages=FINAL_ANSWER_PRE_MESSAGES_PROMPT, post_messages=FINAL_ANSWER_POST_MESSAGES_PROMPT),\n )\n super().__init__(\n *args,\n run_id=run_id,\n context_description=context_description,\n prompt_templates=prompt_templates,\n max_steps=12,\n verbosity_level=2,\n additional_authorized_imports=[\"*\"],\n planning_interval=4,\n return_full_result=True,\n **kwargs,\n )\n\n# class ManagerAgent(CodeAgent):\n\n# def _step_stream(self, memory_step: ActionStep) -> Generator[ChatMessageStreamDelta | ActionOutput | ToolOutput]:\n# \"\"\"\n# Perform one step in the ReAct framework: the agent thinks, acts, and observes the result.\n# Yields ChatMessageStreamDelta during the run if streaming is enabled.\n# At the end, yields either None if the step is not final, or the final answer.\n# \"\"\"\n# raise NotImplementedError(\"This method should be implemented in child classes\")" }, { "path": "src/palimpzest/constants.py", "content": "### This file contains constants used by Palimpzest ###\nfrom __future__ import annotations\n\nimport os\nfrom enum import Enum\n\nimport litellm\n\nfrom palimpzest.utils.model_info_helpers import ModelMetricsManager, predict_local_model_metrics\n\n\nclass PromptStrategy(str, Enum):\n \"\"\"\n PromptStrategy describes the prompting technique to be used by a Generator when\n performing some task with a specified Model.\n \"\"\"\n\n # aggregation prompt strategies\n AGG = \"aggregation\"\n AGG_NO_REASONING = \"aggregation-no-reasoning\"\n\n # filter prompt strategies\n FILTER = \"filter\"\n FILTER_NO_REASONING = \"filter-no-reasoning\"\n FILTER_CRITIC = \"filter-critic\"\n FILTER_REFINE = \"filter-refine\"\n FILTER_MOA_PROPOSER = \"filter-mixture-of-agents-proposer\"\n FILTER_MOA_AGG = \"filter-mixture-of-agents-aggregator\"\n FILTER_SPLIT_PROPOSER = \"filter-split-proposer\"\n FILTER_SPLIT_MERGER = \"filter-split-merger\"\n\n # join prompt strategies\n JOIN = \"join\"\n JOIN_NO_REASONING = \"join-no-reasoning\"\n\n # map prompt strategies\n MAP = \"map\"\n MAP_NO_REASONING = \"map-no-reasoning\"\n MAP_CRITIC = \"map-critic\"\n MAP_REFINE = \"map-refine\"\n MAP_MOA_PROPOSER = \"map-mixture-of-agents-proposer\"\n MAP_MOA_AGG = \"map-mixture-of-agents-aggregator\"\n MAP_SPLIT_PROPOSER = \"map-split-proposer\"\n MAP_SPLIT_MERGER = \"map-split-merger\"\n\n def is_agg_prompt(self):\n return \"aggregation\" in self.value\n\n def is_filter_prompt(self):\n return \"filter\" in self.value\n\n def is_join_prompt(self):\n return \"join\" in self.value\n\n def is_map_prompt(self):\n return \"map\" in self.value\n\n def is_critic_prompt(self):\n return \"critic\" in self.value\n\n def is_refine_prompt(self):\n return \"refine\" in self.value\n\n def is_moa_proposer_prompt(self):\n return \"mixture-of-agents-proposer\" in self.value\n\n def is_moa_aggregator_prompt(self):\n return \"mixture-of-agents-aggregator\" in self.value\n\n def is_split_proposer_prompt(self):\n return \"split-proposer\" in self.value\n\n def is_split_merger_prompt(self):\n return \"split-merger\" in self.value\n\n def is_no_reasoning_prompt(self):\n return \"no-reasoning\" in self.value\n\n\nclass Modality(str, Enum):\n TEXT = \"text\"\n IMAGE = \"image\"\n AUDIO = \"audio\"\n\n\nclass AggFunc(str, Enum):\n COUNT = \"count\"\n AVERAGE = \"average\"\n SUM = \"sum\"\n MIN = \"min\"\n MAX = \"max\"\n\nclass Cardinality(str, Enum):\n ONE_TO_ONE = \"one-to-one\"\n ONE_TO_MANY = \"one-to-many\"\n\n @classmethod\n def _missing_(cls, value):\n if value:\n normalized_value = \"\".join([x for x in value if x.isalpha()]).lower()\n for member in cls:\n normalized_member = \"\".join([x for x in member if x.isalpha()]).lower()\n if normalized_member == normalized_value:\n return member\n return cls.ONE_TO_ONE\n\n\nclass PickOutputStrategy(str, Enum):\n CHAMPION = \"champion\"\n ENSEMBLE = \"ensemble\"\n\n\nAUDIO_EXTENSIONS = [\".wav\"]\nIMAGE_EXTENSIONS = [\".jpg\", \".jpeg\", \".png\", \".gif\", \".bmp\", \".tiff\"]\nPDF_EXTENSIONS = [\".pdf\"]\nXLS_EXTENSIONS = [\".xls\", \".xlsx\"]\nHTML_EXTENSIONS = [\".html\", \".htm\"]\n\n# the number of seconds the parallel execution will sleep for while waiting for futures to complete\nPARALLEL_EXECUTION_SLEEP_INTERVAL_SECS = 0.3\n\n# default PDF parser\nDEFAULT_PDF_PROCESSOR = \"pypdf\"\n\n# character limit for various IDs\nMAX_ID_CHARS = 10\n\n# maximum number of rows to display in a table\nMAX_ROWS = 5\n\n# maximum number of rows to parse from an HTML\nMAX_HTML_ROWS = 10000\n\n\ndef log_attempt_number(retry_state):\n \"\"\"return the result of the last call attempt\"\"\"\n print(f\"Retrying: {retry_state.attempt_number}...\")\n\n\n# Palimpzest root directory\nPZ_DIR = os.path.join(os.path.expanduser(\"~\"), \".palimpzest\")\n\n# assume 500 MB/sec for local SSD scan time\nLOCAL_SCAN_TIME_PER_KB = 1 / (float(500) * 1024)\n\n# assume 30 GB/sec for sequential access of memory\nMEMORY_SCAN_TIME_PER_KB = 1 / (float(30) * 1024 * 1024)\n\n# assume 1 KB per record\nNAIVE_BYTES_PER_RECORD = 1024\n\n# rough conversion from # of characters --> # of tokens; assumes 1 token ~= 4 chars\nTOKENS_PER_CHARACTER = 0.25\n\n# rough estimate of the number of tokens the context is allowed to take up for LLAMA3 models\nLLAMA_CONTEXT_TOKENS_LIMIT = 6000\n\n# a naive estimate for the input record size\nNAIVE_EST_SOURCE_RECORD_SIZE_IN_BYTES = 1_000_000\n\n# a naive estimate for filter selectivity\nNAIVE_EST_FILTER_SELECTIVITY = 0.5\n\n# a naive estimate for join selectivity\nNAIVE_EST_JOIN_SELECTIVITY = 0.5\n\n# a naive estimate for the number of input tokens processed per record\nNAIVE_EST_NUM_INPUT_TOKENS = 1000\n\n# a naive estimate for the number of output tokens processed per record\nNAIVE_EST_NUM_OUTPUT_TOKENS = 100\n\n# a naive estimate for the number of groups returned by a group by\nNAIVE_EST_NUM_GROUPS = 3\n\n# a naive estimate for the factor of increase (loosely termed \"selectivity\") for one-to-many cardinality operations\nNAIVE_EST_ONE_TO_MANY_SELECTIVITY = 2\n\n# a naive estimate of the time it takes to extract the latex for an equation from an image file using Skema\nNAIVE_IMAGE_TO_EQUATION_LATEX_TIME_PER_RECORD = 10.0\n\n# a naive estimate of the time it takes to extract the text from a PDF using a PDF processor\nNAIVE_PDF_PROCESSOR_TIME_PER_RECORD = 10.0\n\n# whether or not to log LLM outputs\nLOG_LLM_OUTPUT = False\n\n# maximum number of models to use when user does not narrow optimization space\nMAX_AVAILABLE_MODELS = 5\n\nclass Model:\n \"\"\"\n Model describes the underlying LLM which should be used to perform some operation\n which requires invoking an LLM.\n \"\"\"\n # Registry of known models (maps value string to Model instance)\n _registry: dict[str, Model] = {}\n\n def __init__(self, model_id: str, api_base: str | None = None, **vllm_kwargs):\n self.metrics_manager = ModelMetricsManager()\n self.model_id = model_id\n self.api_base = api_base\n self.vllm_kwargs = vllm_kwargs if vllm_kwargs else {}\n\n # For vLLM models (api_base is set), try to get model info from litellm's local data\n if api_base is not None:\n self.model_specs = self._get_litellm_model_specs(model_id)\n else:\n self.model_specs = self.metrics_manager.get_model_metrics(model_id)\n if not self.model_specs:\n raise ValueError(\"Palimpzest currently does not contain information about this model.\")\n\n Model._registry[model_id] = self\n\n def _get_litellm_model_specs(self, model_id: str) -> dict:\n \"\"\"Get model specs from litellm's local model_cost data for vLLM models.\"\"\"\n # Use predict function to get quality, latency metrics, and capability flags\n predicted_metrics = predict_local_model_metrics(model_id)\n\n # Start with defaults, then overlay predicted values\n specs = {\n \"is_text_model\": True,\n \"is_vision_model\": False,\n \"is_llama_model\": False,\n \"is_clip_model\": False,\n \"is_audio_model\": False,\n \"is_reasoning_model\": False,\n \"is_embedding_model\": False,\n \"is_text_image_multimodal_embedding_model\": False,\n \"is_vllm_model\": True, # Mark as vLLM model\n \"usd_per_input_token\": 0.0, # Cost always 0 for local model\n \"usd_per_output_token\": 0.0,\n \"seconds_per_output_token\": predicted_metrics[\"seconds_per_output_token\"],\n \"MMLU_Pro_score\": predicted_metrics[\"MMLU_Pro_score\"],\n }\n\n # Overlay all flags detected from model name (including False values like is_text_model for embeddings)\n for key, value in predicted_metrics.items():\n if key.startswith(\"is_\"):\n specs[key] = value\n\n # Try litellm for additional capability detection (may not work for local models)\n try:\n if litellm.supports_vision(model=model_id):\n specs[\"is_vision_model\"] = True\n except Exception:\n pass\n\n try:\n if litellm.supports_audio_input(model=model_id):\n specs[\"is_audio_model\"] = True\n except Exception:\n pass\n\n return specs\n\n def __lt__(self, other):\n if isinstance(other, Model):\n return self.value < other.value\n if isinstance(other, str):\n return self.value < other\n return NotImplemented\n\n @classmethod\n def get_all_models(cls) -> list[Model]:\n return list(cls._registry.values())\n\n @property\n def value(self) -> str:\n return self.model_id\n\n @property\n def provider(self) -> str | None:\n \"\"\"Returns the provider string for this model.\"\"\"\n return self.model_specs.get(\"provider\")\n\n @property\n def api_key_env_var(self) -> str | None:\n \"\"\"\n Returns the standard environment variable name for this provider's API key.\n \"\"\"\n if self.provider == \"gemini\":\n return \"GEMINI_API_KEY\" if os.getenv(\"GEMINI_API_KEY\") else \"GOOGLE_API_KEY\"\n if self.provider == \"azure\":\n return \"AZURE_API_KEY\" if os.getenv(\"AZURE_API_KEY\") else \"AZURE_OPENAI_API_KEY\"\n mapping = {\n \"openai\": \"OPENAI_API_KEY\",\n \"vertex_ai\": \"GOOGLE_APPLICATION_CREDENTIALS\",\n \"anthropic\": \"ANTHROPIC_API_KEY\",\n \"together_ai\": \"TOGETHER_API_KEY\",\n \"hosted_vllm\": \"VLLM_API_KEY\"\n }\n return mapping.get(self.provider)\n\n def __repr__(self) -> str:\n return self.value\n\n def __str__(self) -> str:\n return self.value\n\n def __eq__(self, other: object) -> bool:\n if isinstance(other, Model):\n return self.value == other.value\n if isinstance(other, str):\n return self.value == other\n return NotImplemented\n\n def __hash__(self) -> int:\n return hash(self.value)\n\n def is_llama_model(self) -> bool:\n return self.model_specs.get(\"is_llama_model\", False)\n \n def is_vllm_model(self) -> bool:\n return self.model_specs.get(\"is_vllm_model\", False) and self.api_base is not None\n \n def is_embedding_model(self) -> bool:\n return self.model_specs.get(\"is_embedding_model\", False)\n\n def is_text_image_multimodal_embedding_model(self) -> bool:\n return self.model_specs.get(\"is_text_image_multimodal_embedding_model\", False)\n\n def is_provider_vertex_ai(self) -> bool:\n return self.provider == \"vertex_ai\"\n \n def is_provider_anthropic(self) -> bool:\n return self.provider == \"anthropic\"\n \n def is_provider_google_ai_studio(self) -> bool:\n return self.provider == \"gemini\"\n\n def is_provider_openai(self) -> bool:\n return self.provider == \"openai\"\n \n def is_provider_azure(self) -> bool:\n return self.provider == \"azure\"\n \n def is_provider_together_ai(self) -> bool:\n return self.provider == \"together_ai\"\n \n def is_provider_deepseek(self) -> bool:\n return self.provider == \"deepseek\"\n\n def is_provider_ollama(self) -> bool:\n return self.provider == \"ollama\"\n\n def is_model_gemini(self) -> bool:\n return \"gemini\" in self.value.lower()\n\n def get_model_name(self) -> str:\n return self.value.split(\"/\")[-1] if \"/\" in self.value else self.value\n\n def is_o_model(self) -> bool:\n return self.model_specs.get(\"is_o_model\", False)\n\n def is_gpt_5_model(self) -> bool:\n return self.model_specs.get(\"is_gpt_5_model\", False)\n\n def is_reasoning_model(self) -> bool:\n return self.model_specs.get(\"is_reasoning_model\", False)\n\n def is_text_model(self) -> bool:\n return self.model_specs.get(\"is_text_model\", False)\n\n def is_vision_model(self) -> bool:\n return self.model_specs.get(\"is_vision_model\", False)\n\n def is_audio_model(self) -> bool:\n return self.model_specs.get(\"is_audio_model\", False)\n\n def is_text_image_multimodal_model(self) -> bool:\n return self.is_text_model() and self.is_vision_model()\n\n def is_text_audio_multimodal_model(self) -> bool:\n return self.is_audio_model() and self.is_text_model()\n\n def supports_prompt_caching(self) -> bool:\n return (self.is_provider_anthropic() or self.is_provider_google_ai_studio() or self.is_provider_vertex_ai or self.is_provider_openai() or self.is_provider_azure()) \\\n and self.model_specs.get(\"supports_prompt_caching\", False)\n\n def get_usd_per_input_token(self) -> float:\n return self.model_specs.get(\"usd_per_input_token\", 0.0)\n\n def get_usd_per_audio_input_token(self) -> float:\n return self.model_specs.get(\"usd_per_audio_input_token\", self.get_usd_per_input_token())\n \n # forward-looking, TODO: default value discussion\n def get_usd_per_image_input_token(self) -> float:\n return self.model_specs.get(\"usd_per_image_input_token\", self.get_usd_per_input_token())\n\n def get_usd_per_cache_read_token(self) -> float:\n return self.model_specs.get(\"usd_per_cache_read_token\", self.get_usd_per_input_token())\n \n def get_usd_per_audio_cache_read_token(self) -> float:\n return self.model_specs.get(\"usd_per_audio_cache_read_token\", self.get_usd_per_cache_read_token())\n \n def get_usd_per_image_cache_read_token(self) -> float:\n return self.model_specs.get(\"usd_per_image_cache_read_token\", self.get_usd_per_cache_read_token())\n \n # forward looking; Gemini explicit\n def get_usd_per_cached_token_per_hour(self) -> float:\n return self.model_specs.get(\"usd_per_cached_token_per_hour\", 0.0)\n \n def get_usd_per_cache_creation_token(self) -> float:\n return self.model_specs.get(\"usd_per_cache_creation_token\", 0.0)\n \n def get_usd_per_output_token(self) -> float:\n return self.model_specs.get(\"usd_per_output_token\", 0.0)\n \n # forward-looking\n def get_usd_per_audio_cache_creation_token(self) -> float:\n return self.model_specs.get(\"usd_per_audio_cache_creation_token\", 0.0)\n \n # forward-looking\n def get_usd_per_image_cache_creation_token(self) -> float:\n return self.model_specs.get(\"usd_per_image_cache_creation_token\", 0.0)\n \n def get_seconds_per_output_token(self) -> float:\n return self.model_specs.get(\"seconds_per_output_token\", 0.0)\n\n def get_overall_score(self) -> float:\n return self.model_specs.get(\"MMLU_Pro_score\", 0.0)\n\n# TODO: investigate which (if any llama3 models are still supported by TogetherAI)\n# Model.LLAMA3_2_3B = Model(\"together_ai/meta-llama/Llama-3.2-3B-Instruct-Turbo\") - seems to be deprecated\nModel.LLAMA3_1_8B = Model(\"together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo\")\nModel.LLAMA3_3_70B = Model(\"together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo\")\nModel.LLAMA3_2_90B_V = Model(\"together_ai/meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo\")\nModel.DEEPSEEK_V3 = Model(\"together_ai/deepseek-ai/DeepSeek-V3\")\nModel.DEEPSEEK_R1_DISTILL_QWEN_1_5B = Model(\"together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B\")\nModel.GPT_4o = Model(\"openai/gpt-4o-2024-08-06\")\nModel.GPT_4o_MINI = Model(\"openai/gpt-4o-mini-2024-07-18\")\nModel.GPT_4_1 = Model(\"openai/gpt-4.1-2025-04-14\")\nModel.GPT_4_1_MINI = Model(\"openai/gpt-4.1-mini-2025-04-14\")\nModel.GPT_4_1_NANO = Model(\"openai/gpt-4.1-nano-2025-04-14\")\nModel.GPT_5 = Model(\"openai/gpt-5-2025-08-07\")\nModel.GPT_5_MINI = Model(\"openai/gpt-5-mini-2025-08-07\")\nModel.GPT_5_NANO = Model(\"openai/gpt-5-nano-2025-08-07\")\nModel.GPT_5_2 = Model(\"openai/gpt-5.2-2025-12-11\")\nModel.o4_MINI = Model(\"openai/o4-mini-2025-04-16\") # noqa: N815\n# Model.CLAUDE_3_5_SONNET = Model(\"anthropic/claude-3-5-sonnet-20241022\") - retired 10/28/2025\nModel.CLAUDE_3_7_SONNET = Model(\"anthropic/claude-3-7-sonnet-20250219\")\nModel.CLAUDE_4_SONNET = Model(\"anthropic/claude-sonnet-4-20250514\")\nModel.CLAUDE_4_5_SONNET = Model(\"anthropic/claude-sonnet-4-5-20250929\")\nModel.CLAUDE_3_5_HAIKU = Model(\"anthropic/claude-3-5-haiku-20241022\")\nModel.CLAUDE_4_5_HAIKU = Model(\"anthropic/claude-haiku-4-5-20251001\")\nModel.GEMINI_3_0_PRO = Model(\"vertex_ai/gemini-3-pro-preview\") # image\nModel.GEMINI_3_0_FLASH = Model(\"vertex_ai/gemini-3-flash-preview\") # Text, Image, Video, Audio, and PDF\nModel.GEMINI_2_0_FLASH = Model(\"vertex_ai/gemini-2.0-flash\")\nModel.GEMINI_2_5_FLASH = Model(\"vertex_ai/gemini-2.5-flash\")\nModel.GEMINI_2_5_PRO = Model(\"vertex_ai/gemini-2.5-pro\")\nModel.GOOGLE_GEMINI_3_0_PRO = Model(\"gemini/gemini-3-pro-preview\")\nModel.GOOGLE_GEMINI_3_0_FLASH = Model(\"gemini/gemini-3-flash-preview\")\nModel.GOOGLE_GEMINI_2_5_FLASH = Model(\"gemini/gemini-2.5-flash\")\nModel.GOOGLE_GEMINI_2_5_FLASH_LITE = Model(\"gemini/gemini-2.5-flash-lite\")\nModel.GOOGLE_GEMINI_2_5_PRO = Model(\"gemini/gemini-2.5-pro\")\nModel.LLAMA_4_MAVERICK = Model(\"vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas\")\nModel.GPT_4o_AUDIO_PREVIEW = Model(\"openai/gpt-4o-audio-preview\")\nModel.GPT_4o_MINI_AUDIO_PREVIEW = Model(\"openai/gpt-4o-mini-audio-preview\")\nModel.AZURE_GPT_4o = Model(\"azure/gpt-4o-2024-08-06\")\nModel.AZURE_GPT_4o_MINI = Model(\"azure/gpt-4o-mini-2024-07-18\")\nModel.AZURE_GPT_4_1 = Model(\"azure/gpt-4.1-2025-04-14\")\nModel.AZURE_GPT_4_1_MINI = Model(\"azure/gpt-4.1-mini-2025-04-14\")\nModel.AZURE_GPT_4_1_NANO = Model(\"azure/gpt-4.1-nano-2025-04-14\")\nModel.AZURE_o4_MINI = Model(\"azure/o4-mini-2025-04-16\") # noqa: N815\nModel.AZURE_GPT_4o_AUDIO_PREVIEW = Model(\"azure/gpt-4o-audio-preview\")\nModel.AZURE_GPT_4o_MINI_AUDIO_PREVIEW = Model(\"azure/gpt-4o-mini-audio-preview\")\nModel.TEXT_EMBEDDING_3_SMALL = Model(\"openai/text-embedding-3-small\")\nModel.CLIP_VIT_B_32 = Model(\"clip-ViT-B-32\")\nModel.NOMIC_EMBED_TEXT = Model(\"ollama/nomic-embed-text\")\n\n#### MODEL PERFORMANCE & COST METRICS ####\n# Overall model quality is computed using MMLU-Pro; multi-modal models currently use the same score for vision\n# - in the future we should split quality for vision vs. multi-modal vs. text\n# - code quality was computed using HumanEval, but that benchmark is too easy and should be replaced.\n# - https://huggingface.co/spaces/TIGER-Lab/MMLU-Pro\n# - https://www.vals.ai/benchmarks/mmlu_pro-08-12-2025\n#\n# Cost is presented in terms of USD / token for input tokens and USD / token for\n# generated tokens.\n#\n# Time is presented in seconds per output token. I grabbed some semi-recent estimates\n# from the internet for this quick POC, but we can and should do more to model these\n# values more precisely:\n# - https://artificialanalysis.ai/models/llama-3-1-instruct-8b\n# \n# Model metrics now fetched from singular json file curated_model_info.json\n" }, { "path": "src/palimpzest/core/__init__.py", "content": "" }, { "path": "src/palimpzest/core/data/__init__.py", "content": "" }, { "path": "src/palimpzest/core/data/context.py", "content": "from __future__ import annotations\n\nimport os\nimport re\nfrom abc import ABC\nfrom typing import Callable\n\nfrom pydantic import BaseModel\nfrom smolagents import CodeAgent, LiteLLMModel\n\nfrom palimpzest.core.data import context_manager\nfrom palimpzest.core.data.dataset import Dataset\nfrom palimpzest.core.lib.schemas import create_schema_from_fields, union_schemas\nfrom palimpzest.query.operators.logical import ComputeOperator, ContextScan, LogicalOperator, SearchOperator\nfrom palimpzest.utils.hash_helpers import hash_for_id\n\nPZ_INSTRUCTION = \"\"\"\\n\\nYou are a CodeAgent who is a specialist at writing declarative AI programs with the Palimpzest (PZ) library.\n\nPalimpzest is a programming framework which provides you with **semantic operators** (e.g. semantic maps, semantic filters, etc.)\nwhich are like their traditional counterparts, except they can execute instructions provided in natural language.\n\nFor example, if you wanted to write a program to extract the title and abstract from a directory of papers,\nyou could write the following in PZ:\n```\nimport palimpzest as pz\nfrom dotenv import load_dotenv\n\n# Load environment variables from .env file\nload_dotenv()\n\n# Define columns for semantic map (sem_map) operation; each column is specified\n# with a dictionary containing the following keys:\n# - \"name\": the name of the field to compute\n# - \"type\": the type of the field to compute\n# - \"description\": the natural language description of the field\npaper_cols = [\n {\"name\": \"title\", \"type\": str, \"description\": \"the title of the paper\"},\n {\"name\": \"abstract\", \"type\": str, \"description\": \"the paper's abstract\"},\n]\n\n# construct the data processing pipeline with PZ\nds = pz.TextFileDataset(id=\"papers\", path=\"path/to/papers\")\nds = ds.sem_map(cols)\n\n# optimize and execute the PZ program\nvalidator = pz.Validator()\nconfig = pz.QueryProcessorConfig(\n policy=pz.MaxQuality(),\n execution_strategy=\"parallel\",\n max_workers=20,\n progress=True,\n)\noutput = ds.optimize_and_run(config=config, validator=validator)\n\n# write the execution stats to json\noutput.execution_stats.to_json(\"pz_program_stats.json\")\n\n# write the output to a CSV and print the output CSV filepath so the user knows where to find it\noutput_filepath = \"pz_program_output.csv\"\noutput.to_df().to_csv(output_filepath, index=False)\nprint(f\"Results at: {output_filepath}\")\n```\n\nTo initialize a dataset in PZ, simply provide the path to a directory to `pz.TextFileDirectory()`\n(if your data contains text-based files). For example:\n```\nimport palimpzest as pz\nfrom dotenv import load_dotenv\n\n# Load environment variables from .env file\nload_dotenv()\n\nds = pz.TextFileDataset(id=\"files\", path=\"path/to/files\")\n```\n\nPalimpzest has two primary **semantic operators** which you can use to construct data processing pipelines:\n- sem_filter(predicate: str): executes a semantic filter specified by the natural language predicate on a given PZ dataset\n- sem_map(cols: list[dict]): executes a semantic map to compute the `cols` on a given PZ dataset\n\nAs a second example, consider the following PZ program which filters for papers about batteries that are from MIT\nand computes a summary for each one:\n```\nimport palimpzest as pz\nfrom dotenv import load_dotenv\n\n# Load environment variables from .env file\nload_dotenv()\n\n# construct the PZ program\nds = pz.TextFileDataset(id=\"papers\", path=\"path/to/research-papers\")\nds = ds.sem_filter(\"The paper is about batteries\")\nds = ds.sem_filter(\"The paper is from MIT\")\nds = ds.sem_map([{\"name\": \"summary\", \"type\": str, \"description\": \"A summary of the paper\"}])\n\n# optimize and execute the PZ program\nvalidator = pz.Validator()\nconfig = pz.QueryProcessorConfig(\n policy=pz.MaxQuality(),\n execution_strategy=\"parallel\",\n max_workers=20,\n progress=True,\n)\noutput = ds.optimize_and_run(config=config, validator=validator)\n\n# write the execution stats to json\noutput.execution_stats.to_json(\"pz_program_stats.json\")\n\n# write the output to a CSV and print the output CSV filepath so the user knows where to find it\noutput_filepath = \"pz_program_output.csv\"\noutput.to_df().to_csv(output_filepath, index=False)\nprint(f\"Results at: {output_filepath}\")\n```\n\nBe sure to always:\n- execute your program using the `.optimize_and_run()` format shown above\n- call `output.execution_stats.to_json(\"pz_program_stats.json\")` to write execution statistics to disk\n- write your output to CSV and print where you wrote it!\n\"\"\"\n\nclass Context(Dataset, ABC):\n \"\"\"\n The `Context` class is an abstract base class for root `Datasets` whose data is accessed\n via user-defined methods. Classes which inherit from this class must implement two methods:\n\n - `list_filepaths()`: which lists the files that the `Context` has access to.\n - `read_filepath(path: str)`: which reads the file corresponding to the given `path`.\n\n A `Context` is a special type of `Dataset` that represents a view over an underlying `Dataset`.\n Each `Context` has a `name` which uniquely identifies it, as well as a natural language `description`\n of the data / computation that the `Context` represents. Similar to `Dataset`s, `Context`s can be\n lazily transformed using functions such as `sem_filter`, `sem_map`, `sem_join`, etc., and they may\n be materialized or unmaterialized.\n \"\"\"\n\n def __init__(\n self,\n id: str,\n description: str,\n operator: LogicalOperator,\n schema: type[BaseModel] | None = None,\n sources: list[Context] | Context | None = None,\n materialized: bool = False,\n ) -> None:\n \"\"\"\n Constructor for the `Context` class.\n\n Args:\n id (`str`): a string identifier for the `Context`\n description (`str`): the description of the data contained within the `Context`\n operator (`LogicalOperator`): The `LogicalOperator` used to compute this `Context`.\n schema: (`type[BaseModel] | None`): The schema of this `Context`.\n sources (`list[Context] | Context | None`): The (list of) `Context(s)` which are input(s) to\n the operator used to compute this `Context`.\n materialized (`bool`): True if the `Context` has been computed, False otherwise\n \"\"\"\n # set the description\n self._description = description\n\n # set the materialization status\n self._materialized = materialized\n\n # compute schema and call parent constructor\n if schema is None:\n schema = create_schema_from_fields([{\"name\": \"context\", \"description\": \"The context\", \"type\": str}])\n super().__init__(sources=sources, operator=operator, schema=schema, id=id)\n\n # set the tools associated with this Context\n self._tools = [getattr(self, attr) for attr in dir(self) if attr.startswith(\"tool_\")]\n\n # add Context to ContextManager\n cm = context_manager.ContextManager()\n cm.add_context(self)\n\n @property\n def description(self) -> str:\n \"\"\"The string containing all of the information computed for this `Context`\"\"\"\n return self._description\n\n @property\n def materialized(self) -> bool:\n \"\"\"The boolean which specifies whether the `Context` has been computed or not\"\"\"\n return self._materialized\n\n @property\n def tools(self) -> list[Callable]:\n \"\"\"The list of tools associated with this `Context`\"\"\"\n return self._tools\n\n def __str__(self) -> str:\n return f\"Context(id={self.id}, description={self.description:20s}, materialized={self.materialized})\"\n\n def set_description(self, description: str) -> None:\n \"\"\"\n Update the context's description.\n \"\"\"\n self._description = description\n\n def set_materialized(self, materialized: str) -> None:\n \"\"\"\n Update the context's materialization status.\n \"\"\"\n self._materialized = materialized\n\n def compute(self, instruction: str) -> Context:\n # construct new description and output schema\n new_id = hash_for_id(instruction)\n new_description = f\"Parent Context ID: {self.id}\\n\\nThis Context is the result of computing the following instruction on the parent context.\\n\\nINSTRUCTION: {instruction}\\n\\n\"\n inter_schema = create_schema_from_fields([{\"name\": f\"result-{new_id}\", \"desc\": \"The result from computing the instruction on the input Context\", \"type\": str}])\n new_output_schema = union_schemas([self.schema, inter_schema])\n\n # construct logical operator\n operator = ComputeOperator(\n input_schema=self.schema,\n output_schema=new_output_schema,\n context_id=new_id,\n instruction=instruction,\n ) \n\n return Context(id=new_id, description=new_description, operator=operator, sources=[self], materialized=False)\n\n def search(self, search_query: str) -> Context:\n # construct new description and output schema\n new_id = hash_for_id(search_query)\n new_description = f\"Parent Context ID: {self.id}\\n\\nThis Context is the result of searching the parent context for information related to the following query.\\n\\nSEARCH QUERY: {search_query}\\n\\n\"\n\n # construct logical operator\n operator = SearchOperator(\n input_schema=self.schema,\n output_schema=self.schema,\n context_id=new_id,\n search_query=search_query,\n )\n\n return Context(id=new_id, description=new_description, operator=operator, sources=[self], materialized=False)\n\nclass TextFileContext(Context):\n def __init__(self, path: str, id: str, description: str) -> None:\n \"\"\"\n Constructor for the `TextFileContext` class.\n\n Args:\n path (str): The path to the file\n id (str): a string identifier for the `Context`\n description (str): The description of the data contained within the `Context`\n kwargs (dict): Keyword arguments containing the `Context's` id and description.\n \"\"\"\n # check that path is a valid file or directory\n assert os.path.isfile(path) or os.path.isdir(path), f\"Path {path} is not a file nor a directory\"\n\n # get list of filepaths\n self.filepaths = []\n if os.path.isfile(path):\n self.filepaths = [path]\n else:\n self.filepaths = []\n for root, _, files in os.walk(path):\n for file in files:\n fp = os.path.join(root, file)\n self.filepaths.append(fp)\n self.filepaths = sorted(self.filepaths)\n\n # call parent constructor to set id, operator, and schema\n schema = create_schema_from_fields([{\"name\": \"context\", \"desc\": \"The context\", \"type\": str}])\n super().__init__(\n id=id,\n description=description,\n operator=ContextScan(context=self, output_schema=schema),\n schema=schema,\n materialized=True,\n )\n def _check_filter_answer_text(self, answer_text: str) -> dict | None:\n \"\"\"\n Return {\"passed_operator\": True} if and only if \"true\" is in the answer text.\n Return {\"passed_operator\": False} if and only if \"false\" is in the answer text.\n Otherwise, return None.\n \"\"\"\n # NOTE: we may be able to eliminate this condition by specifying this JSON output in the prompt;\n # however, that would also need to coincide with a change to allow the parse_answer_fn to set \"passed_operator\"\n if \"true\" in answer_text.lower():\n return {\"passed_operator\": True}\n elif \"false\" in answer_text.lower():\n return {\"passed_operator\": False}\n elif \"yes\" in answer_text.lower():\n return {\"passed_operator\": True}\n\n return None\n\n def _parse_filter_answer(self, completion_text: str) -> dict[str, list]:\n \"\"\"Extract the answer from the completion object for filter operations.\"\"\"\n # if the model followed the default instructions, the completion text will place\n # its answer between \"ANSWER:\" and \"---\"\n regex = re.compile(\"answer:(.*?)---\", re.IGNORECASE | re.DOTALL)\n matches = regex.findall(completion_text)\n if len(matches) > 0:\n answer_text = matches[0].strip()\n field_answers = self._check_filter_answer_text(answer_text)\n if field_answers is not None:\n return field_answers\n\n # if the first regex didn't find an answer, try taking all the text after \"ANSWER:\"\n regex = re.compile(\"answer:(.*)\", re.IGNORECASE | re.DOTALL)\n matches = regex.findall(completion_text)\n if len(matches) > 0:\n answer_text = matches[0].strip()\n field_answers = self._check_filter_answer_text(answer_text)\n if field_answers is not None:\n return field_answers\n\n # finally, try taking all of the text; throw an exception if this doesn't work\n field_answers = self._check_filter_answer_text(completion_text)\n if field_answers is None:\n raise Exception(f\"Could not parse answer from completion text: {completion_text}\")\n\n return field_answers\n\n # def tool_list_filepaths(self) -> list[str]:\n # \"\"\"\n # This tool returns the list of all of the filepaths which the `Context` has access to.\n\n # Args:\n # None\n \n # Returns:\n # list[str]: A list of file paths for all files in the `Context`.\n # \"\"\"\n # return self.filepaths\n\n # def tool_read_filepath(self, path: str) -> str:\n # \"\"\"\n # This tool takes a filepath (`path`) as input and returns the content of the file as a string.\n # It handles both CSV files and html / regular text files. It does not handle images.\n\n # Args:\n # path (str): The path to the file to read.\n\n # Returns:\n # str: The content of the file as a string.\n # \"\"\"\n # if path.endswith(\".csv\"):\n # return pd.read_csv(path, encoding=\"ISO-8859-1\").to_string(index=False)\n\n # with open(path, encoding='utf-8') as file:\n # content = file.read()\n\n # return content\n\n def tool_execute_semantic_operators(self, instruction: str) -> str:\n \"\"\"\n This tool takes an `instruction` as input and invokes an expert to write a semantic data processing pipeline\n to execute the instruction. The tool returns the path to a CSV file which contains the output of the pipeline.\n\n For example, the tool could be invoked as follows to extract the title and abstract from a dataset of research papers:\n ```\n instruction = \"Write a program to extract the title and abstract from each research paper\"\n result_csv_filepath = tool_execute_semantic_operators(instruction)\n ```\n\n Args:\n instruction: The instruction specifying the semantic data processing pipeline that you need to execute.\n\n Returns:\n str: the filepath to the CSV containing the output from running the data processing pipeline.\n \"\"\"\n from smolagents import tool\n @tool\n def tool_list_filepaths() -> list[str]:\n \"\"\"\n This tool returns the list of all of the filepaths which the `Context` has access to.\n\n NOTE: You may want to execute this before writing your PZ program to determine where the data lives.\n\n Args:\n None\n \n Returns:\n list[str]: A list of file paths for all files in the `Context`.\n \"\"\"\n return self.filepaths\n\n agent = CodeAgent(\n model=LiteLLMModel(model_id=\"openai/o1\", api_key=os.getenv(\"ANTHROPIC_API_KEY\")),\n tools=[tool_list_filepaths],\n max_steps=20,\n planning_interval=4,\n add_base_tools=False,\n return_full_result=True,\n additional_authorized_imports=[\"dotenv\", \"json\", \"palimpzest\", \"pandas\"],\n instructions=PZ_INSTRUCTION,\n )\n result = agent.run(instruction)\n response = result.output\n\n return response\n" }, { "path": "src/palimpzest/core/data/context_manager.py", "content": "from __future__ import annotations\n\nimport os\nimport pickle\n\nimport chromadb\nimport chromadb.utils.embedding_functions as embedding_functions\nimport tiktoken\n\nfrom palimpzest.constants import PZ_DIR\nfrom palimpzest.core.data import context\n\n\nclass ContextNotFoundError(Exception):\n pass\n\n\nclass ContextManager:\n \"\"\"\n This class manages the long-term storage of `Contexts`. Each new `Context` is added to\n the `ContextManager` and serialized to disk. `Contexts` are also indexed, which enables\n PZ to search for `Context(s)` which may support `search()` and `compute()` operations.\n \"\"\"\n def __init__(self):\n # create directory with serialized contexts (if it doesn't already exist)\n self.context_dir = os.path.join(PZ_DIR, \"contexts\")\n os.makedirs(self.context_dir, exist_ok=True)\n\n # create vector store (if it doesn't already exist)\n self.chroma_dir = os.path.join(PZ_DIR, \"chroma\")\n os.makedirs(self.chroma_dir, exist_ok=True)\n self.chroma_client = chromadb.PersistentClient(self.chroma_dir)\n\n # pick embedding function based on presence of API key(s)\n self.emb_fn = None\n if os.getenv(\"OPENAI_API_KEY\"):\n self.emb_fn = embedding_functions.OpenAIEmbeddingFunction(\n api_key=os.getenv(\"OPENAI_API_KEY\"),\n model_name=\"text-embedding-3-small\"\n )\n\n self.index = self.chroma_client.get_or_create_collection(\"contexts\", embedding_function=self.emb_fn)\n\n @staticmethod\n def from_pkl(path: str) -> context.Context:\n \"\"\"Load a `Context` from its serialized pickle file.\"\"\"\n with open(path, \"rb\") as f:\n context = pickle.load(f)\n\n return context\n\n @staticmethod\n def to_pkl(context: context.Context, path: str) -> None:\n \"\"\"Write the given `Context` to a pickle file at the provided `path`.\"\"\"\n with open(path, \"wb\") as f:\n pickle.dump(context, f)\n\n def num_tokens_from_string(self, string: str, encoding_name: str) -> int:\n \"\"\"Returns the number of tokens in a text string.\"\"\"\n encoding = tiktoken.get_encoding(encoding_name)\n num_tokens = len(encoding.encode(string))\n return num_tokens\n\n def add_context(self, context: context.Context, update: bool = False) -> None:\n \"\"\"\n Add the new `Context` to the `ContextManager` by serializing and writing it to disk.\n\n Args:\n context (`Context`): the context to add to the `ContextManager`\n update (`bool`): whether or not to update an existing context\n\n TODO: track cost\n \"\"\"\n # return early if the context already exists and we're not performing an update\n id = context.id\n context_path = os.path.join(self.context_dir, f\"{id}.pkl\")\n if os.path.exists(context_path) and update is False:\n return\n\n # write the context to disk\n ContextManager.to_pkl(context, context_path)\n\n # compute number of tokens in context.description\n description = context.description\n while self.num_tokens_from_string(description, \"cl100k_base\") > 8192:\n description = description[:int(0.9*len(description))]\n \n # add context to vector store\n context_embeddings = self.emb_fn([description])\n context_payload = {\n \"ids\": [context.id],\n \"embeddings\": context_embeddings,\n \"metadatas\": [{\"id\": context.id, \"materialized\": context.materialized}],\n \"documents\": [context.description],\n }\n if update:\n self.index.update(**context_payload)\n else:\n self.index.add(**context_payload)\n\n def update_context(self, id: str, description: str, materialized: bool = True) -> None:\n \"\"\"\n Update an existing `Context` with the given `id` to have the given `description`.\n \n Args:\n id (str): the id of the updated `Context`\n description (str): the update to the description for the specified `Context`\n materialized (bool): boolean to set the materialization status of the `Context`\n\n Raises:\n ContextNotFoundError: if the given `id` doesn't point to a `Context` in the `ContextManger`.\n \"\"\"\n context = self.get_context(id)\n new_description = context.description + description # TODO: should description have RESULT replaced on update? as opposed to appending? should description be some pydantic BaseModel?\n context.set_description(new_description)\n context.set_materialized(materialized)\n self.add_context(context, update=True)\n\n def get_context(self, id: str) -> context.Context:\n \"\"\"\n Returns the `Context` specified by the given `id`.\n\n Args:\n id (str): the id of the retrieved `Context`\n\n Returns:\n `Context`: the specified `Context`.\n \"\"\"\n context_path = os.path.join(self.context_dir, f\"{id}.pkl\")\n try:\n return ContextManager.from_pkl(context_path)\n except FileNotFoundError as err:\n raise ContextNotFoundError from err\n\n def search_context(self, query: str, k: int = 1, where: dict | None = None) -> list[context.Context]:\n \"\"\"\n Returns the top-k most relevant `Context(s)` for the given query. If provided,\n the where dictionary will be used to filter the search results.\n\n TODO:\n 3) update CostModel to account for benefit of using existing Context(s)\n ---\n 4) unit test\n 5) track cost\n \"\"\"\n # embed the search query\n query_embeddings = self.emb_fn([query])\n\n # look up ids of most similar contexts\n results = self.index.query(\n query_embeddings=query_embeddings,\n n_results=k,\n where=where,\n )\n ids = results[\"ids\"][0]\n\n # load and return Context objects\n contexts = []\n for id in ids:\n context_path = os.path.join(self.context_dir, f\"{id}.pkl\")\n contexts.append(ContextManager.from_pkl(context_path))\n\n return contexts\n" }, { "path": "src/palimpzest/core/data/dataset.py", "content": "from __future__ import annotations\n\nimport warnings\nfrom collections.abc import Iterator\nfrom typing import Callable\n\nfrom chromadb.api.models.Collection import Collection\nfrom pydantic import BaseModel\n\nfrom palimpzest.constants import AggFunc, Cardinality\nfrom palimpzest.core.elements.filters import Filter\nfrom palimpzest.core.elements.groupbysig import GroupBySig\nfrom palimpzest.core.lib.schemas import create_schema_from_fields, project, relax_schema, union_schemas\nfrom palimpzest.policy import construct_policy_from_kwargs\nfrom palimpzest.query.operators.logical import (\n Aggregate,\n ConvertScan,\n Distinct,\n FilteredScan,\n GroupByAggregate,\n JoinOp,\n LimitScan,\n LogicalOperator,\n Project,\n TopKScan,\n)\nfrom palimpzest.query.processor.config import QueryProcessorConfig\nfrom palimpzest.utils.hash_helpers import hash_for_serialized_dict\nfrom palimpzest.validator.validator import Validator\n\n\n# TODO?: remove `schema` from `Dataset` and access it from `operator`?\n# - Q: how do you handle datasets with multiple sources?\n# - for joins the operator should have the union'ed schema\n# - but for Contexts it may be trickier\nclass Dataset:\n \"\"\"\n A `Dataset` represents a collection of structured or unstructured data that can be processed and\n transformed. Each `Dataset` is either a \"root\" `Dataset` (which yields data items) or it is the\n result of performing data processing operation(s) on root `Dataset(s)`.\n\n Users can perform computations on a `Dataset` in a lazy or eager fashion. Applying functions\n such as `sem_filter`, `sem_map`, `sem_join`, `sem_agg`, etc. will lazily create a new `Dataset`.\n Users can invoke the `run()` method to execute the computation and retrieve a materialized `Dataset`.\n Materialized `Dataset`s can be processed further, or their results can be retrieved using `.get()`.\n \n A root `Dataset` must subclass at least one of `pz.IterDataset`, `pz.IndexDataset`, or `pz.Context`.\n Each of these classes supports a different access pattern:\n\n - `pz.IterDataset`: supports accessing data via iteration\n - Ex: iterating over a list of PDFs\n - Ex: iterating over rows in a DataFrame\n - `pz.IndexDataset`: supports accessing data via point lookups / queries\n - Ex: querying a vector database\n - Ex: querying a SQL database\n - `pz.Context`: supports accessing data with an agent\n - Ex: processing a set of CSV files with a data science agent\n - Ex: processing time series data with a data cleaning agent\n\n A root `Dataset` may subclass more than one of the aforementioned classes. For example, the root\n `Dataset` for a list of files may inherit from `pz.IterDataset` and `pz.IndexDataset` to support\n iterating over the files and performing point lookups for individual files.\n\n For details on how to create your own root `Dataset`, please see: TODO\n \"\"\"\n def __init__(\n self,\n sources: list[Dataset] | Dataset | None,\n operator: LogicalOperator,\n schema: type[BaseModel] | None = None,\n id: str | None = None,\n ) -> None:\n \"\"\"\n Initialize a `Dataset` with one or more `sources` and the operator that is being applied.\n Root `Datasets` subclass `pz.IterDataset`, `pz.IndexDataset`, and/or `pz.Context` and use\n their own constructors.\n\n Args:\n sources (`list[Dataset] | Dataset`): The (list of) `Dataset(s)` which are input(s) to\n the operator used to compute this `Dataset`.\n operator (`LogicalOperator`): The `LogicalOperator` used to compute this `Dataset`.\n schema (type[`BaseModel`] | None): The schema of this `Dataset`.\n id (str | None): an identifier for this `Dataset` provided by the user\n\n Raises:\n ValueError: if `sources` is not a `Dataset` or list of `Datasets`\n \"\"\"\n # set sources\n self._sources = []\n if isinstance(sources, list):\n self._sources = sources\n elif isinstance(sources, Dataset):\n self._sources = [sources]\n elif sources is not None:\n raise ValueError(\"Dataset sources must be another Dataset or a list of Datasets. For root Datasets, you must subclass pz.IterDataset, pz.IndexDataset, or pz.Context.\")\n\n # set the logical operator and schema\n self._operator: LogicalOperator = operator\n self._schema = schema\n\n # compute the dataset id\n self._id = self._compute_dataset_id() if id is None else id\n\n @property\n def id(self) -> str:\n \"\"\"The string identifier for this `Dataset`\"\"\"\n return self._id\n\n @property\n def schema(self) -> type[BaseModel]:\n \"\"\"The Pydantic model defining the schema of this `Dataset`\"\"\"\n return self._schema\n\n @property\n def is_root(self) -> bool:\n return len(self._sources) == 0\n\n def __str__(self) -> str:\n return f\"Dataset(schema={self._schema}, id={self._id}, op_id={self._operator.get_logical_op_id()})\"\n\n def __iter__(self) -> Iterator[Dataset]:\n for source in self._sources:\n yield from source\n yield self\n\n def _compute_dataset_id(self) -> str:\n \"\"\"\n Compute the identifier for this `Dataset`. The ID is uniquely defined by the operation(s)\n applied to the `Dataset's` sources.\n \"\"\"\n return hash_for_serialized_dict({\n \"source_ids\": [source.id for source in self._sources],\n \"logical_op_id\": self._operator.get_logical_op_id(),\n })\n\n def _set_root_datasets(self, new_root_datasets: dict[str, Dataset]) -> None:\n \"\"\"\n Update the root dataset(s) for this dataset with the `new_root_datasets`. This is used during\n optimization to reuse the same physical plan while running it on a train dataset.\n\n Args:\n new_root_datasets (dict[str, Dataset]): the new root datasets for this dataset.\n \"\"\"\n new_sources = []\n for old_source in self._sources:\n if old_source.id in new_root_datasets:\n new_sources.append(new_root_datasets[old_source.id])\n else:\n old_source._set_root_datasets(new_root_datasets)\n new_sources.append(old_source)\n self._sources = new_sources\n\n # TODO: the entire way (unique) logical op ids are computed and stored needs to be rethought\n def _generate_unique_logical_op_ids(self, topo_idx: int | None = None) -> None:\n \"\"\"\n Generate unique operation IDs for all operators in this dataset and its sources.\n This is used to ensure that each operator can be uniquely identified during execution.\n \"\"\"\n # generate the unique op ids for all sources' operators\n for source in self._sources:\n topo_idx = source._generate_unique_logical_op_ids(topo_idx)\n topo_idx += 1\n\n # if topo_idx is None, this is the first call, so we initialize it to 0\n if topo_idx is None:\n topo_idx = 0\n\n # compute this operator's unique operator id\n this_unique_logical_op_id = f\"{topo_idx}-{self._operator.get_logical_op_id()}\"\n\n # update the unique logical op id for this operator\n self._operator.set_unique_logical_op_id(this_unique_logical_op_id)\n\n # return the current unique full_op_id for this operator\n return topo_idx\n\n # TODO\n def _resolve_depends_on(self, depends_on: list[str]) -> list[str]:\n \"\"\"\n TODO: resolve the `depends_on` strings to their full field names ({Dataset.id}.{field_name}).\n \"\"\"\n return []\n\n def _get_root_datasets(self) -> dict[str, Dataset]:\n \"\"\"Return a mapping from the id --> Dataset for all root datasets.\"\"\"\n if self.is_root:\n return {self.id: self}\n\n root_datasets = {}\n for source in self._sources:\n child_root_datasets = source._get_root_datasets()\n root_datasets = {**root_datasets, **child_root_datasets}\n\n return root_datasets\n\n def relax_types(self) -> None:\n \"\"\"\n Relax the types in this Dataset's schema and all upstream Datasets' schemas to be more permissive.\n \"\"\"\n # relax the types in this dataset's schema\n self._schema = relax_schema(self._schema)\n\n # relax the types in dataset's operator's input and output schemas\n self._operator.input_schema = None if self._operator.input_schema is None else relax_schema(self._operator.input_schema)\n self._operator.output_schema = relax_schema(self._operator.output_schema)\n\n # recursively relax the types in all upstream datasets\n for source in self._sources:\n source.relax_types()\n\n def get_upstream_datasets(self) -> list[Dataset]:\n \"\"\"\n Get the list of all upstream datasets that are sources to this dataset.\n \"\"\"\n # recursively get the upstream datasets\n upstream = []\n for source in self._sources:\n upstream.extend(source.get_upstream_datasets())\n upstream.append(source)\n return upstream\n\n def get_limit(self) -> int | None:\n \"\"\"Get the limit applied to this Dataset, if any.\"\"\"\n if isinstance(self._operator, LimitScan):\n return self._operator.limit\n\n source_limits = []\n for source in self._sources:\n source_limit = source.get_limit()\n if source_limit is not None:\n source_limits.append(source_limit)\n\n if len(source_limits) == 0:\n return None\n\n return min([limit for limit in source_limits if limit is not None])\n\n def copy(self):\n return Dataset(\n sources=[source.copy() for source in self._sources],\n operator=self._operator.copy(),\n schema=self._schema,\n id=self.id,\n )\n\n def join(self, other: Dataset, on: str | list[str], how: str = \"inner\") -> Dataset:\n \"\"\"\n Perform the specified join on the specified (list of) column(s)\n \"\"\"\n # enforce type for on\n if isinstance(on, str):\n on = [on]\n\n # construct new output schema\n combined_schema = union_schemas([self.schema, other.schema], join=True, on=on)\n\n # construct logical operator\n operator = JoinOp(\n input_schema=combined_schema,\n output_schema=combined_schema,\n condition=\"\",\n on=on,\n how=how,\n depends_on=on,\n )\n\n return Dataset(sources=[self, other], operator=operator, schema=combined_schema)\n\n def sem_join(self, other: Dataset, condition: str, desc: str | None = None, depends_on: str | list[str] | None = None, how: str = \"inner\") -> Dataset:\n \"\"\"\n Perform a semantic (inner) join on the specified join predicate\n \"\"\"\n # enforce type for depends_on\n if isinstance(depends_on, str):\n depends_on = [depends_on]\n\n # construct new output schema\n combined_schema = union_schemas([self.schema, other.schema], join=True)\n\n # construct logical operator\n operator = JoinOp(\n input_schema=combined_schema,\n output_schema=combined_schema,\n condition=condition,\n how=how,\n desc=desc,\n depends_on=depends_on,\n )\n\n return Dataset(sources=[self, other], operator=operator, schema=combined_schema)\n\n def filter(\n self,\n filter: Callable,\n depends_on: str | list[str] | None = None,\n ) -> Dataset:\n \"\"\"Add a user defined function as a filter to the Set. This filter will possibly restrict the items that are returned later.\"\"\"\n # construct Filter object\n f = None\n if callable(filter):\n f = Filter(filter_fn=filter)\n else:\n error_str = f\"Only support callable for filter, currently got {type(filter)}\"\n if isinstance(filter, str):\n error_str += \". Consider using sem_filter() for semantic filters.\"\n raise Exception(error_str)\n\n # enforce type for depends_on\n if isinstance(depends_on, str):\n depends_on = [depends_on]\n\n # construct logical operator\n operator = FilteredScan(input_schema=self.schema, output_schema=self.schema, filter=f, depends_on=depends_on)\n\n return Dataset(sources=[self], operator=operator, schema=self.schema)\n\n def sem_filter(\n self,\n filter: str,\n desc: str | None = None,\n depends_on: str | list[str] | None = None,\n ) -> Dataset:\n \"\"\"Add a natural language description of a filter to the Set. This filter will possibly restrict the items that are returned later.\"\"\"\n # construct Filter object\n f = None\n if isinstance(filter, str):\n f = Filter(filter)\n else:\n raise Exception(\"sem_filter() only supports `str` input for _filter.\", type(filter))\n\n # enforce type for depends_on\n if isinstance(depends_on, str):\n depends_on = [depends_on]\n\n # construct logical operator\n operator = FilteredScan(input_schema=self.schema, output_schema=self.schema, filter=f, desc=desc, depends_on=depends_on)\n\n return Dataset(sources=[self], operator=operator, schema=self.schema)\n\n def _sem_map(self, cols: list[dict] | type[BaseModel] | None,\n cardinality: Cardinality,\n desc: str | None = None,\n depends_on: str | list[str] | None = None) -> Dataset:\n \"\"\"Execute the semantic map operation with the appropriate cardinality.\"\"\"\n # construct new output schema\n new_output_schema = None\n if cols is None:\n new_output_schema = self.schema\n elif isinstance(cols, list):\n cols = create_schema_from_fields(cols)\n new_output_schema = union_schemas([self.schema, cols])\n elif issubclass(cols, BaseModel):\n new_output_schema = union_schemas([self.schema, cols])\n else:\n raise ValueError(\"`cols` must be a list of dictionaries or a BaseModel.\")\n\n # enforce type for depends_on\n if isinstance(depends_on, str):\n depends_on = [depends_on]\n\n # construct logical operator\n operator = ConvertScan(\n input_schema=self.schema,\n output_schema=new_output_schema,\n cardinality=cardinality,\n udf=None,\n desc=desc,\n depends_on=depends_on,\n )\n\n return Dataset(sources=[self], operator=operator, schema=new_output_schema)\n\n def sem_add_columns(self, cols: list[dict] | type[BaseModel],\n cardinality: Cardinality = Cardinality.ONE_TO_ONE,\n desc: str | None = None,\n depends_on: str | list[str] | None = None) -> Dataset:\n \"\"\"\n NOTE: we are renaming this function to `sem_map` and deprecating `sem_add_columns` in the next\n release of PZ. To update your code, simply change your calls from `.sem_add_columns(...)` to `.sem_map(...)`.\n The function arguments will stay the same.\n\n Add new columns by specifying the column names, descriptions, and types.\n The column will be computed during the execution of the Dataset.\n Example:\n sem_add_columns(\n [{'name': 'greeting', 'desc': 'The greeting message', 'type': str},\n {'name': 'age', 'desc': 'The age of the person', 'type': int},\n {'name': 'full_name', 'desc': 'The name of the person', 'type': str}]\n )\n \"\"\"\n # issue deprecation warning\n warnings.warn(\n \"we are renaming this function to `sem_map` and deprecating `sem_add_columns` in the next\"\n \" release of PZ. To update your code, simply change your calls from `.sem_add_columns(...)`\"\n \" to `.sem_map(...)`. The function arguments will stay the same.\",\n DeprecationWarning,\n stacklevel=2\n )\n\n return self._sem_map(cols, cardinality, desc, depends_on)\n\n def sem_map(self, cols: list[dict] | type[BaseModel], desc: str | None = None, depends_on: str | list[str] | None = None) -> Dataset:\n \"\"\"\n Compute new field(s) by specifying their names, descriptions, and types. For each input there will\n be one output. The field(s) will be computed during the execution of the Dataset.\n\n Example:\n sem_map(\n [{'name': 'greeting', 'desc': 'The greeting message', 'type': str},\n {'name': 'age', 'desc': 'The age of the person', 'type': int},\n {'name': 'full_name', 'desc': 'The name of the person', 'type': str}]\n )\n \"\"\"\n return self._sem_map(cols, Cardinality.ONE_TO_ONE, desc, depends_on)\n\n def sem_flat_map(self, cols: list[dict] | type[BaseModel], desc: str | None = None, depends_on: str | list[str] | None = None) -> Dataset:\n \"\"\"\n Compute new field(s) by specifying their names, descriptions, and types. For each input there will\n be one or more output(s). The field(s) will be computed during the execution of the Dataset.\n\n Example:\n sem_flat_map(\n cols=[\n {'name': 'author_name', 'description': 'The name of the author', 'type': str},\n {'name': 'institution', 'description': 'The institution of the author', 'type': str},\n {'name': 'email', 'description': 'The author's email', 'type': str},\n ]\n )\n \"\"\"\n return self._sem_map(cols, Cardinality.ONE_TO_MANY, desc, depends_on)\n\n def _map(self, udf: Callable,\n cols: list[dict] | type[BaseModel] | None,\n cardinality: Cardinality,\n depends_on: str | list[str] | None = None) -> Dataset:\n \"\"\"Execute the map operation with the appropriate cardinality.\"\"\"\n # construct new output schema\n new_output_schema = None\n if cols is None:\n new_output_schema = self.schema\n elif isinstance(cols, list):\n cols = create_schema_from_fields(cols)\n new_output_schema = union_schemas([self.schema, cols])\n elif issubclass(cols, BaseModel):\n new_output_schema = union_schemas([self.schema, cols])\n else:\n raise ValueError(\"`cols` must be a list of dictionaries, a BaseModel, or None.\")\n\n # enforce type for depends_on\n if isinstance(depends_on, str):\n depends_on = [depends_on]\n\n # construct logical operator\n operator = ConvertScan(\n input_schema=self.schema,\n output_schema=new_output_schema,\n cardinality=cardinality,\n udf=udf,\n depends_on=depends_on,\n )\n\n return Dataset(sources=[self], operator=operator, schema=new_output_schema)\n\n def add_columns(self, udf: Callable,\n cols: list[dict] | type[BaseModel] | None,\n cardinality: Cardinality = Cardinality.ONE_TO_ONE,\n depends_on: str | list[str] | None = None) -> Dataset:\n \"\"\"\n NOTE: we are renaming this function to `map` and deprecating `add_columns` in the next\n release of PZ. To update your code, simply change your calls from `.add_columns(...)` to `.map(...)`.\n The function arguments will stay the same.\n\n Compute new fields (or update existing ones) with a UDF. For each input, this function will compute one output.\n\n Set `cols=None` if your add_columns operation is not computing any new fields.\n\n Examples:\n add_columns(\n udf=compute_personal_greeting,\n cols=[\n {'name': 'greeting', 'description': 'The greeting message', 'type': str},\n {'name': 'age', 'description': 'The age of the person', 'type': int},\n {'name': 'full_name', 'description': 'The name of the person', 'type': str},\n ]\n )\n \"\"\"\n # issue deprecation warning\n warnings.warn(\n \"we are renaming this function to `map` and deprecating `add_columns` in the next\"\n \" release of PZ. To update your code, simply change your calls from `.add_columns(...)`\"\n \" to `.map(...)`. The function arguments will stay the same.\",\n DeprecationWarning,\n stacklevel=2\n )\n\n # sanity check inputs\n if udf is None:\n raise ValueError(\"`udf` must be provided for add_columns.\")\n\n return self._map(udf, cols, cardinality, depends_on)\n\n def map(self, udf: Callable,\n cols: list[dict] | type[BaseModel] | None,\n depends_on: str | list[str] | None = None) -> Dataset:\n \"\"\"\n Compute new fields (or update existing ones) with a UDF. For each input, this function will compute one output.\n\n Set `cols=None` if your map is not computing any new fields.\n\n Examples:\n map(\n udf=compute_personal_greeting,\n cols=[\n {'name': 'greeting', 'description': 'The greeting message', 'type': str},\n {'name': 'age', 'description': 'The age of the person', 'type': int},\n {'name': 'full_name', 'description': 'The name of the person', 'type': str},\n ]\n )\n \"\"\"\n # sanity check inputs\n if udf is None:\n raise ValueError(\"`udf` must be provided for map.\")\n\n return self._map(udf, cols, Cardinality.ONE_TO_ONE, depends_on)\n\n def flat_map(self, udf: Callable,\n cols: list[dict] | type[BaseModel] | None,\n depends_on: str | list[str] | None = None) -> Dataset:\n \"\"\"\n Compute new fields (or update existing ones) with a UDF. For each input, this function will compute one or more outputs.\n\n Set `cols=None` if your flat_map is not computing any new fields.\n\n Examples:\n flat_map(\n udf=extract_paper_authors,\n cols=[\n {'name': 'author_name', 'description': 'The name of the author', 'type': str},\n {'name': 'institution', 'description': 'The institution of the author', 'type': str},\n {'name': 'email', 'description': 'The author's email', 'type': str},\n ]\n )\n \"\"\"\n # sanity check inputs\n if udf is None:\n raise ValueError(\"`udf` must be provided for map.\")\n\n return self._map(udf, cols, Cardinality.ONE_TO_MANY, depends_on)\n\n def count(self) -> Dataset:\n \"\"\"Apply a count aggregation to this set\"\"\"\n operator = Aggregate(input_schema=self.schema, agg_func=AggFunc.COUNT)\n return Dataset(sources=[self], operator=operator, schema=operator.output_schema)\n\n def average(self) -> Dataset:\n \"\"\"Apply an average aggregation to this set\"\"\"\n operator = Aggregate(input_schema=self.schema, agg_func=AggFunc.AVERAGE)\n return Dataset(sources=[self], operator=operator, schema=operator.output_schema)\n\n def sum(self) -> Dataset:\n \"\"\"Apply a summation to this set\"\"\"\n operator = Aggregate(input_schema=self.schema, agg_func=AggFunc.SUM)\n return Dataset(sources=[self], operator=operator, schema=operator.output_schema)\n\n def min(self) -> Dataset:\n \"\"\"Apply an min operator to this set\"\"\"\n operator = Aggregate(input_schema=self.schema, agg_func=AggFunc.MIN)\n return Dataset(sources=[self], operator=operator, schema=operator.output_schema)\n\n def max(self) -> Dataset:\n \"\"\"Apply an max operator to this set\"\"\"\n operator = Aggregate(input_schema=self.schema, agg_func=AggFunc.MAX)\n return Dataset(sources=[self], operator=operator, schema=operator.output_schema)\n\n def groupby(self, groupby: GroupBySig) -> Dataset:\n output_schema = groupby.output_schema()\n operator = GroupByAggregate(input_schema=self.schema, output_schema=output_schema, group_by_sig=groupby)\n return Dataset(sources=[self], operator=operator, schema=output_schema)\n\n def sem_agg(self, col: dict | type[BaseModel], agg: str, depends_on: str | list[str] | None = None) -> Dataset:\n \"\"\"\n Apply a semantic aggregation to this set. The `agg` string will be applied using an LLM\n over the entire set of inputs' fields specified in `depends_on` to generate the output `col`.\n\n Example:\n sem_agg(\n col={'name': 'overall_sentiment', 'desc': 'The overall sentiment of the reviews', 'type': str},\n agg=\"Compute the overall sentiment of the reviews as POSITIVE or NEGATIVE.\",\n depends_on=\"review_text\",\n )\n \"\"\"\n # construct new output schema\n new_output_schema = None\n if isinstance(col, dict):\n new_output_schema = create_schema_from_fields([col])\n elif issubclass(col, BaseModel):\n assert len(col.model_fields) == 1, \"For semantic aggregation, when passing a BaseModel to `col` it must have exactly one field.\"\n new_output_schema = col\n else:\n raise ValueError(\"`col` must be a dictionary or a single-field BaseModel.\")\n\n # enforce type for depends_on\n if isinstance(depends_on, str):\n depends_on = [depends_on]\n\n # construct logical operator\n operator = Aggregate(input_schema=self.schema, output_schema=new_output_schema, agg_str=agg, depends_on=depends_on)\n\n return Dataset(sources=[self], operator=operator, schema=operator.output_schema)\n\n def sem_topk(\n self,\n index: Collection,\n search_attr: str,\n output_attrs: list[dict] | type[BaseModel],\n search_func: Callable | None = None,\n k: int = -1,\n ) -> Dataset:\n \"\"\"\n Retrieve the top-k nearest neighbors of the value of the `search_attr` from the `index` and\n use these results to construct the `output_attrs` field(s).\n \"\"\"\n # construct new output schema\n new_output_schema = None\n if isinstance(output_attrs, list):\n output_attrs = create_schema_from_fields(output_attrs)\n new_output_schema = union_schemas([self.schema, output_attrs])\n elif issubclass(output_attrs, BaseModel):\n new_output_schema = union_schemas([self.schema, output_attrs])\n else:\n raise ValueError(\"`output_attrs` must be a list of dictionaries or a BaseModel.\")\n\n # TODO: revisit once we can think through abstraction(s)\n # # construct the PZIndex from the user-provided index\n # index = index_factory(index)\n\n # construct logical operator\n operator = TopKScan(\n input_schema=self.schema,\n output_schema=new_output_schema,\n index=index,\n search_func=search_func,\n search_attr=search_attr,\n output_attrs=output_attrs,\n k=k,\n )\n\n return Dataset(sources=[self], operator=operator, schema=new_output_schema)\n\n def limit(self, n: int) -> Dataset:\n \"\"\"Limit the set size to no more than n rows\"\"\"\n operator = LimitScan(input_schema=self.schema, output_schema=self.schema, limit=n)\n return Dataset(sources=[self], operator=operator, schema=self.schema)\n\n def distinct(self, distinct_cols: list[str] | None = None) -> Dataset:\n \"\"\"Return a new Dataset with distinct rows based on the current schema.\"\"\"\n operator = Distinct(input_schema=self.schema, output_schema=self.schema, distinct_cols=distinct_cols)\n return Dataset(sources=[self], operator=operator, schema=self.schema)\n\n def project(self, project_cols: list[str] | str) -> Dataset:\n \"\"\"Project the Set to only include the specified columns.\"\"\"\n project_cols = project_cols if isinstance(project_cols, list) else [project_cols]\n new_output_schema = project(self.schema, project_cols)\n operator = Project(input_schema=self.schema, output_schema=new_output_schema, project_cols=project_cols)\n return Dataset(sources=[self], operator=operator, schema=new_output_schema)\n\n def run(self, config: QueryProcessorConfig | None = None, **kwargs):\n \"\"\"Invoke the QueryProcessor to execute the query. `kwargs` will be applied to the QueryProcessorConfig.\"\"\"\n # TODO: this import currently needs to be here to avoid a circular import; we should fix this in a subsequent PR\n from palimpzest.query.processor.query_processor_factory import QueryProcessorFactory\n\n # as syntactic sugar, we will allow some keyword arguments to parameterize our policies\n policy = construct_policy_from_kwargs(**kwargs)\n if policy is not None:\n kwargs[\"policy\"] = policy\n\n # construct unique logical op ids for all operators in this dataset\n self._generate_unique_logical_op_ids()\n\n return QueryProcessorFactory.create_and_run_processor(self, config)\n\n def optimize_and_run(self, config: QueryProcessorConfig | None = None, train_dataset: dict[str, Dataset] | Dataset | None = None, validator: Validator | None = None, **kwargs):\n \"\"\"Optimize the PZ program using the train_dataset and validator before running the optimized plan.\"\"\"\n # TODO: this import currently needs to be here to avoid a circular import; we should fix this in a subsequent PR\n from palimpzest.query.processor.query_processor_factory import QueryProcessorFactory\n\n # confirm that either train_dataset or validator is provided\n assert train_dataset is not None or validator is not None, \"Must provide at least one of train_dataset or validator to use optimize_and_run()\"\n\n # validate the train_dataset has one input for each source dataset and normalize its type to be a dict\n if train_dataset is not None:\n root_datasets = self._get_root_datasets()\n if isinstance(train_dataset, Dataset) and len(root_datasets) > 1:\n raise ValueError(\n \"For plans with more than one root dataset, `train_dataset` must be a dictionary mapping\"\n \" {'dataset_id' --> Dataset} for all root Datasets\"\n )\n\n elif isinstance(train_dataset, Dataset):\n root_dataset_id = list(root_datasets.values())[0].id\n if train_dataset.id != root_dataset_id:\n warnings.warn(\n f\"train_dataset.id={train_dataset.id} does not match root dataset id={root_dataset_id}\\n\"\n f\"Setting train_dataset to be the training data for root dataset with id={root_dataset_id} anyways.\",\n stacklevel=2,\n )\n train_dataset = {root_dataset_id: train_dataset}\n\n elif not all(dataset_id in train_dataset for dataset_id in root_datasets):\n missing_ids = [dataset_id for dataset_id in root_datasets if dataset_id not in train_dataset]\n raise ValueError(\n f\"`train_dataset` is missing the following root dataset id(s): {missing_ids}\"\n )\n\n # as syntactic sugar, we will allow some keyword arguments to parameterize our policies\n policy = construct_policy_from_kwargs(**kwargs)\n if policy is not None:\n kwargs[\"policy\"] = policy\n config.policy = policy\n\n # construct unique logical op ids for all operators in this dataset\n self._generate_unique_logical_op_ids()\n\n return QueryProcessorFactory.create_and_run_processor(self, config, train_dataset, validator)\n" }, { "path": "src/palimpzest/core/data/index_dataset.py", "content": "from __future__ import annotations\n\nfrom abc import ABC, abstractmethod\n\nfrom chromadb.api.models.Collection import Collection\n\n\ndef index_factory(index: Collection) -> PZIndex:\n \"\"\"\n Factory function to create a PZ index based on the type of the provided index.\n\n Args:\n index (Collection): The index provided by the user.\n\n Returns:\n PZIndex: The PZ wrapped Index.\n \"\"\"\n if isinstance(index, Collection):\n return ChromaIndex(index)\n else:\n raise TypeError(f\"Unsupported index type: {type(index)}\\nindex must be a `chromadb.api.models.Collection.Collection`\")\n\n\nclass BaseIndex(ABC):\n\n def __init__(self, index: Collection):\n self.index = index\n\n def __str__(self):\n \"\"\"\n Return a string representation of the index.\n \"\"\"\n return f\"{self.__class__.__name__}\"\n\n @abstractmethod\n def search(self, query_embedding: list[float] | list[list[float]], results_per_query: int = 1) -> list | list[list]:\n \"\"\"\n Query the index with a string or a list of strings.\n\n Args:\n query (str | list[str]): The query string or list of strings to search for.\n results_per_query (int): The number of top results to retrieve for each query.\n\n Returns:\n list | list[list]: The top results for the query. If query is a list, the top\n results for each query in the list are returned. Each list will contain the\n raw elements yielded by the index. This way, users can program against the\n results they expect to get from e.g. chromadb or ragatouille.\n \"\"\"\n pass\n\n\nclass ChromaIndex(BaseIndex):\n def __init__(self, index: Collection):\n assert isinstance(index, Collection), \"ChromaIndex input must be a `chromadb.api.models.Collection.Collection`\"\n super().__init__(index)\n\n\n# define type for PZIndex\nPZIndex = ChromaIndex\n" }, { "path": "src/palimpzest/core/data/iter_dataset.py", "content": "from __future__ import annotations\n\nimport base64\nimport os\nfrom abc import ABC, abstractmethod\nfrom io import BytesIO\nfrom pathlib import Path\n\nimport pandas as pd\nfrom bs4 import BeautifulSoup\nfrom pydantic import BaseModel\n\nfrom palimpzest import constants\nfrom palimpzest.core.data import dataset\nfrom palimpzest.core.lib.schemas import (\n AudioFile,\n DefaultSchema,\n ImageFile,\n PDFFile,\n TextFile,\n WebPage,\n XLSFile,\n create_schema_from_df,\n create_schema_from_fields,\n)\nfrom palimpzest.query.operators.logical import BaseScan\nfrom palimpzest.tools.pdfparser import get_text_from_pdf\n\n\n####################\n### BASE CLASSES ###\n####################\nclass IterDataset(dataset.Dataset, ABC):\n \"\"\"\n The `IterDataset` is an abstract base class for root `Datasets` whose data is accessed\n via iteration. Classes which inherit from this class must implement two methods:\n\n - `__len__()`: which returns the number of elements in the dataset\n - `__getitem__(idx: int)`: which takes in an `idx` and returns the element at that index\n \"\"\"\n\n def __init__(self, id: str, schema: type[BaseModel] | list[dict]) -> None:\n \"\"\"\n Constructor for the `IterDataset` class.\n\n Args:\n id (str): a string identifier for the `Dataset`\n schema (BaseModel | list[dict]): The output schema of the records returned by the `Dataset`\n \"\"\"\n # compute Schema and call parent constructor\n schema = create_schema_from_fields(schema) if isinstance(schema, list) else schema\n super().__init__(sources=None, operator=BaseScan(datasource=self, output_schema=schema), schema=schema, id=id)\n\n @abstractmethod\n def __len__(self) -> int:\n \"\"\"Returns the number of items in the `Dataset`.\"\"\"\n pass\n\n @abstractmethod\n def __getitem__(self, idx: int) -> dict:\n \"\"\"\n Returns a single item from the `Dataset` at the given index.\n\n Args:\n idx (int): The index of the item to return\n\n Returns:\n dict: A dictionary representing the item at the given index. The dictionary\n keys (i.e. fields) should match the fields specified in the schema of the\n dataset, and the values should be the values associated with those fields.\n\n # Example return value\n {\"field1\": value1, \"field2\": value2, ...}\n\n \"\"\"\n pass\n\n\nclass BaseFileDataset(IterDataset):\n \"\"\"\n BaseFileDataset is the base class for multiple `IterDatasets` which iterate over\n different types of files.\n \"\"\"\n\n def __init__(self, path: str, **kwargs) -> None:\n \"\"\"\n Constructor for the `BaseFileDataset` class.\n\n Args:\n path (str): The path to the file\n kwargs (dict): Keyword arguments containing the `Dataset's` id and file-specific `Schema`\n \"\"\"\n # check that path is a valid file or directory\n assert os.path.isfile(path) or os.path.isdir(path), f\"Path {path} is not a file nor a directory\"\n\n # get list of filepaths\n self.filepaths = []\n if os.path.isfile(path):\n self.filepaths = [path]\n else:\n self.filepaths = [\n os.path.join(path, filename)\n for filename in sorted(os.listdir(path))\n if os.path.isfile(os.path.join(path, filename))\n ]\n\n # call parent constructor to set id, operator, and schema\n super().__init__(**kwargs)\n\n def __len__(self) -> int:\n return len(self.filepaths)\n\n\nclass BaseFileDirectoryDataset(IterDataset):\n \"\"\"\n BaseFileDirectoryDataset is the base class for multiple `IterDatasets` which iterate over\n different types of files. This class walks the entire directory tree rooted at `path`.\n \"\"\"\n\n def __init__(self, path: str, **kwargs) -> None:\n \"\"\"\n Constructor for the `BaseFileDataset` class.\n\n Args:\n path (str): The path to the file\n kwargs (dict): Keyword arguments containing the `Dataset's` id and file-specific `Schema`\n \"\"\"\n # check that path is a valid file or directory\n assert os.path.isfile(path) or os.path.isdir(path), f\"Path {path} is not a file nor a directory\"\n\n # get list of filepaths\n self.filepaths = []\n if os.path.isfile(path):\n self.filepaths = [path]\n else:\n self.filepaths = []\n for root, _, files in os.walk(path):\n for file in files:\n fp = os.path.join(root, file)\n self.filepaths.append(fp)\n self.filepaths = sorted(self.filepaths)\n\n # call parent constructor to set id, operator, and schema\n super().__init__(**kwargs)\n\n def __len__(self) -> int:\n return len(self.filepaths)\n\n########################\n### CONCRETE CLASSES ###\n########################\nclass MemoryDataset(IterDataset):\n \"\"\"\n MemoryDataset returns one or more dictionaries that reflect the contents of an in-memory Python object `vals`.\n If `vals` is not a pd.DataFrame, then the dictionary returned by `__getitem__()` has a single field called \"value\".\n Otherwise, the dictionary contains the key-value mapping from columns to values for the `idx` row in the dataframe.\n\n TODO(gerardo): Add support for other types of in-memory data structures (he has some code for subclassing\n MemoryDataset on his branch)\n \"\"\"\n\n def __init__(self, id: str, vals: list | pd.DataFrame, schema: type[BaseModel] | list[dict] | None = None) -> None:\n \"\"\"\n Constructor for the `MemoryDataset` class. The `schema` is set to the default `DefaultSchema` schema.\n If `vals` is a pd.DataFrame, then the schema is set to the schema inferred from the DataFrame.\n\n Args:\n id (str): a string identifier for the `Dataset`\n vals (Any): The in-memory data to iterate over\n \"\"\"\n # if list[dict] --> convert to pd.DataFrame first\n self.vals = pd.DataFrame(vals) if isinstance(vals, list) and all([isinstance(item, dict) for item in vals]) else vals\n if schema is None:\n schema = create_schema_from_df(self.vals) if isinstance(self.vals, pd.DataFrame) else DefaultSchema\n super().__init__(id=id, schema=schema)\n\n def __len__(self) -> int:\n return len(self.vals)\n\n def __getitem__(self, idx: int) -> dict:\n \"\"\"\n Returns a dictionary with the value(s) for the element at the specified `idx` in `vals`.\n\n Args:\n idx (int): The index of the item to return\n\n Returns:\n dict: If `vals` is not a pd.DataFrame, then the dictionary has a single field called \"value\".\n Otherwise, the dictionary contains the key-value mapping from columns to values for the\n `idx` row in the dataframe.\n \n .. code-block:: python\n \n # Example return value at idx = 0, for the following list of values\n # [42, 43, 44, ...]\n {\"value\": 42}\n\n # Example return value at idx = 0, for the following DataFrame:\n # +---------+---------+---------+\n # | name | job | hobby |\n # +---------+---------+---------+\n # | Alice | doctor | tennis |\n # | Bob | lawyer | chess |\n # +---------+---------+---------+\n {\"name\": \"Alice\", \"job\": \"doctor\", \"hobby\": \"tennis\"}\n \"\"\"\n item = (\n self.vals.iloc[idx].to_dict()\n if isinstance(self.vals, pd.DataFrame)\n else {\"value\": self.vals[idx]}\n )\n\n return item\n\n\nclass HTMLFileDataset(BaseFileDataset):\n \"\"\"\n HTMLFileDataset returns a dictionary for each HTML file in a directory. Each dictionary contains the\n filename, raw HTML content, and parsed content of a single HTML file in the directory.\n \"\"\"\n def __init__(self, id: str, path: str) -> None:\n \"\"\"\n Constructor for the `HTMLFileDataset` class. The `schema` is set to the `WebPage` schema.\n\n Args:\n id (str): a string identifier for the `Dataset`\n path (str): The path to the directory\n \"\"\"\n super().__init__(path=path, id=id, schema=WebPage)\n self.filepaths = [fp for fp in self.filepaths if fp.endswith(tuple(constants.HTML_EXTENSIONS))]\n\n def _html_to_text_with_links(self, html: str) -> str:\n # Parse the HTML content\n soup = BeautifulSoup(html, \"html.parser\")\n\n # Find all hyperlink tags\n for a in soup.find_all(\"a\"):\n # Check if the hyperlink tag has an 'href' attribute\n if a.has_attr(\"href\"):\n # Replace the hyperlink with its text and URL in parentheses\n a.replace_with(f\"{a.text} ({a['href']})\")\n\n # Extract text from the modified HTML\n text = soup.get_text(separator=\"\\n\", strip=True)\n\n return text\n\n def __getitem__(self, idx: int) -> dict:\n \"\"\"\n Returns a dictionary with the filename, raw HTML content, and parsed content of the HTML file at the\n specified `idx`.\n\n Args:\n idx (int): The index of the item to return\n \n Returns:\n dict: A dictionary with the filename, raw HTML content, and parsed content of the HTML file.\n\n .. code-block:: python\n\n {\n \"filename\": \"file.html\",\n \"html\": \"raw HTML content here\",\n \"text\": \"parsed text content here\",\n }\n \"\"\"\n item = {}\n filepath = self.filepaths[idx]\n item[\"filename\"] = os.path.basename(filepath)\n with open(filepath) as f:\n text_content = f.read()\n\n html = text_content\n tokens = html.split()[: constants.MAX_HTML_ROWS]\n item[\"html\"] = \" \".join(tokens)\n\n stripped_html = self._html_to_text_with_links(text_content)\n tokens = stripped_html.split()[: constants.MAX_HTML_ROWS]\n item[\"text\"] = \" \".join(tokens)\n\n return item\n\n\nclass ImageFileDataset(BaseFileDataset):\n \"\"\"\n ImageFileDataset returns a dictionary for each image file in a directory. Each dictionary contains the\n filename and the base64 encoded bytes content of a single image file in the directory.\n \"\"\"\n def __init__(self, id: str, path: str) -> None:\n \"\"\"\n Constructor for the `ImageFileDataset` class. The `schema` is set to the `ImageFile` schema.\n\n Args:\n id (str): a string identifier for the `Dataset`\n path (str): The path to the directory\n \"\"\"\n super().__init__(path=path, id=id, schema=ImageFile)\n self.filepaths = [fp for fp in self.filepaths if fp.endswith(tuple(constants.IMAGE_EXTENSIONS))]\n\n def __getitem__(self, idx: int) -> dict:\n \"\"\"\n Returns a dictionary with the filename and base64 encoded bytes content of the image file at the\n specified `idx`.\n\n Args:\n idx (int): The index of the item to return\n\n Returns:\n dict: A dictionary with the filename and base64 encoded bytes content of the image file.\n\n .. code-block:: python\n\n {\n \"filename\": \"image.jpg\",\n \"contents\": b\"base64 encoded image content here\",\n }\n \"\"\"\n filepath = self.filepaths[idx]\n filename = os.path.basename(filepath)\n with open(filepath, \"rb\") as f:\n contents = base64.b64encode(f.read()).decode(\"utf-8\")\n\n return {\"filename\": filename, \"contents\": contents}\n\n\nclass PDFFileDataset(BaseFileDataset):\n \"\"\"\n PDFFileDataset returns a dictionary for each PDF file in a directory. Each dictionary contains the\n filename, raw PDF content, and parsed text content of a single PDF file in the directory.\n\n This class also uses one of a predefined set of PDF processors to extract text content from the PDF files.\n \"\"\"\n def __init__(\n self,\n id: str,\n path: str,\n pdfprocessor: str = \"pypdf\",\n file_cache_dir: str = \"/tmp\",\n ) -> None:\n \"\"\"\n Constructor for the `PDFFileDataset` class. The `schema` is set to the `PDFFile` schema.\n\n Args:\n id (str): a string identifier for the `Dataset`\n path (str): The path to the directory\n pdfprocessor (str): The PDF processor to use for extracting text content from the PDF files\n file_cache_dir (str): The directory to store the temporary files generated during PDF processing\n \"\"\"\n super().__init__(path=path, id=id, schema=PDFFile)\n self.filepaths = [fp for fp in self.filepaths if fp.endswith(tuple(constants.PDF_EXTENSIONS))]\n self.pdfprocessor = pdfprocessor\n self.file_cache_dir = file_cache_dir\n\n def __getitem__(self, idx: int) -> dict:\n \"\"\"\n Returns a dictionary with the filename, raw PDF content, and parsed text content of the PDF file at the\n specified `idx`.\n\n Args:\n idx (int): The index of the item to return\n\n Returns:\n dict: A dictionary with the filename, raw PDF content, and parsed text content of the PDF file.\n\n .. code-block:: python\n\n {\n \"filename\": \"file.pdf\",\n \"contents\": b\"raw PDF content here\",\n \"text_contents\": \"parsed text content here\",\n }\n \"\"\"\n filepath = self.filepaths[idx]\n pdf_filename = os.path.basename(filepath)\n with open(filepath, \"rb\") as f:\n pdf_bytes = f.read()\n\n # generate text_content from PDF\n text_content = get_text_from_pdf(pdf_filename, pdf_bytes, pdfprocessor=self.pdfprocessor, file_cache_dir=self.file_cache_dir)\n\n # construct and return item\n return {\"filename\": pdf_filename, \"contents\": pdf_bytes, \"text_contents\": text_content}\n\n\nclass TextFileDataset(BaseFileDataset):\n \"\"\"\n TextFileDataset returns a dictionary for each text file in a directory. Each dictionary contains the\n filename and contents of a single text file in the directory.\n \"\"\"\n def __init__(self, id: str, path: str) -> None:\n \"\"\"\n Constructor for the `TextFileDataset` class. The `schema` is set to the `TextFile` schema.\n\n Args:\n id (str): a string identifier for the `Dataset`\n path (str): The path to the directory\n \"\"\"\n super().__init__(path=path, id=id, schema=TextFile)\n\n def __getitem__(self, idx: int) -> dict:\n \"\"\"\n Returns a dictionary with the filename and contents of the text file at the specified `idx`.\n\n Args:\n idx (int): The index of the item to return\n\n Returns:\n dict: A dictionary with the filename and contents of the text file.\n\n .. code-block:: python\n\n {\n \"filename\": \"file.txt\",\n \"contents\": \"text content here\",\n }\n \"\"\"\n filepath = self.filepaths[idx]\n filename = os.path.basename(filepath)\n with open(filepath) as f:\n contents = f.read()\n\n return {\"filename\": filename, \"contents\": contents}\n\n\nclass XLSFileDataset(BaseFileDataset):\n \"\"\"\n XLSFileDataset returns a dictionary for each XLS file in a directory. Each dictionary contains the\n filename, contents, sheet names, and the number of sheets for a single XLS file in the directory.\n \"\"\"\n def __init__(self, id: str, path: str) -> None:\n \"\"\"\n Constructor for the `XLSFileDataset` class. The `schema` is set to the `XLSFile` schema.\n \"\"\"\n super().__init__(path=path, id=id, schema=XLSFile)\n self.filepaths = [fp for fp in self.filepaths if fp.endswith(tuple(constants.XLS_EXTENSIONS))]\n\n def __getitem__(self, idx: int) -> dict:\n \"\"\"\n Returns a dictionary with the filename, contents, sheet names, and the number of sheets of the XLS file at the\n specified `idx`.\n\n Args:\n idx (int): The index of the item to return\n\n Returns:\n dict: A dictionary with the filename, contents, sheet names, and the number of sheets of the XLS file.\n\n .. code-block:: python\n\n {\n \"filename\": \"file.xls\",\n \"contents\": b\"raw XLS content here\",\n \"sheet_names\": [\"Sheet1\", \"Sheet2\", \"Sheet3],\n \"number_sheets\": 3,\n }\n \"\"\"\n filepath = self.filepaths[idx]\n filename = os.path.basename(filepath)\n with open(filepath, \"rb\") as f:\n contents = f.read()\n\n xls = pd.ExcelFile(BytesIO(contents), engine=\"openpyxl\")\n\n return {\n \"filename\": filename,\n \"contents\": contents,\n \"sheet_names\": xls.sheet_names,\n \"number_sheets\": len(xls.sheet_names),\n }\n\n\nclass AudioFileDataset(BaseFileDirectoryDataset):\n \"\"\"\n AudioFileDataset returns a dictionary for each audio file in a directory. Each dictionary contains the\n filename and the base64 encoded bytes content of a single audio file in the directory.\n \"\"\"\n def __init__(self, id: str, path: str) -> None:\n \"\"\"\n Constructor for the `AudioFileDataset` class. The `schema` is set to the `AudioFile` schema.\n\n Args:\n id (str): a string identifier for the `Dataset`\n path (str): The path to the directory\n \"\"\"\n super().__init__(path=path, id=id, schema=AudioFile)\n self.filepaths = [fp for fp in self.filepaths if fp.endswith(tuple(constants.AUDIO_EXTENSIONS))]\n\n def __getitem__(self, idx: int) -> dict:\n \"\"\"\n Returns a dictionary with the filename and base64 encoded bytes content of the audio file at the\n specified `idx`.\n\n Args:\n idx (int): The index of the item to return\n\n Returns:\n dict: A dictionary with the filename and base64 encoded bytes content of the audio file.\n\n .. code-block:: python\n\n {\n \"filename\": \"audio.wav\",\n \"contents\": b\"base64 encoded audio content here\",\n }\n \"\"\"\n filepath = self.filepaths[idx]\n filename = os.path.basename(filepath)\n with open(filepath, \"rb\") as f:\n contents = base64.b64encode(f.read()).decode(\"utf-8\")\n\n return {\"filename\": filename, \"contents\": contents}\n\n\ndef get_local_source(id: str, path: str | Path, **kwargs) -> dataset.Dataset:\n \"\"\"Return a `Dataset` for a local file or directory.\"\"\"\n if os.path.isfile(path):\n return TextFileDataset(id, path)\n\n elif os.path.isdir(path):\n if all([f.endswith(tuple(constants.IMAGE_EXTENSIONS)) for f in os.listdir(path)]):\n return ImageFileDataset(id, path)\n\n elif all([f.endswith(tuple(constants.PDF_EXTENSIONS)) for f in os.listdir(path)]):\n pdfprocessor = kwargs.get(\"pdfprocessor\", constants.DEFAULT_PDF_PROCESSOR)\n file_cache_dir = kwargs.get(\"file_cache_dir\", \"/tmp\")\n return PDFFileDataset(\n id=id, path=path, pdfprocessor=pdfprocessor, file_cache_dir=file_cache_dir\n )\n\n elif all([f.endswith(tuple(constants.XLS_EXTENSIONS)) for f in os.listdir(path)]):\n return XLSFileDataset(id, path)\n\n elif all([f.endswith(tuple(constants.HTML_EXTENSIONS)) for f in os.listdir(path)]):\n return HTMLFileDataset(id, path)\n\n else:\n return TextFileDataset(id, path)\n else:\n raise ValueError(f\"Path {path} is invalid. Does not point to a file or directory.\")\n\n\ndef resolve_datasource(id: str, source: str | Path | list | pd.DataFrame, **kwargs) -> dataset.Dataset:\n \"\"\"\n This helper function returns a `Dataset` object based on the `source` type.\n The returned `Dataset` object is guaranteed to have a schema.\n \"\"\"\n if isinstance(source, (str, Path)):\n source = get_local_source(id, source, **kwargs)\n\n elif isinstance(source, (list, pd.DataFrame)):\n source = MemoryDataset(id=id, vals=source)\n\n else:\n raise ValueError(f\"Invalid source type: {type(source)}, We only support str, Path, list[dict], and pd.DataFrame\")\n\n return source\n" }, { "path": "src/palimpzest/core/elements/__init__.py", "content": "" }, { "path": "src/palimpzest/core/elements/filters.py", "content": "from __future__ import annotations\n\nfrom typing import Any, Callable\n\n\n#############################\n# Filters that can be applied against a particular Schema\n#############################\n# TODO: think through a way to give filter functions fixed strings that could not be affected by a copy\n# potentially changing the address of a function; I don't think this happens today, but it's worth safeguarding against\nclass Filter:\n \"\"\"A filter that can be applied to a Set\"\"\"\n\n def __init__(self, filter_condition: str | None = None, filter_fn: Callable | None = None) -> None:\n self.filter_condition = filter_condition\n self.filter_fn = filter_fn\n\n def serialize(self) -> dict[str, Any]:\n return {\n \"filter_condition\": self.filter_condition,\n \"filter_fn\": self.filter_fn.__name__ if self.filter_fn is not None else None,\n }\n\n def get_filter_str(self) -> str:\n return self.filter_condition if self.filter_condition is not None else self.filter_fn.__name__\n\n def __repr__(self) -> str:\n return \"Filter(\" + self.get_filter_str() + \")\"\n\n def __hash__(self) -> int:\n # custom hash function\n return hash(self.filter_condition) if self.filter_condition is not None else hash(self.filter_fn.__name__)\n\n def __eq__(self, other) -> bool:\n # __eq__ should be defined for consistency with __hash__\n return (\n isinstance(other, Filter)\n and self.filter_condition == other.filter_condition\n and self.filter_fn == other.filter_fn\n )\n\n def __str__(self) -> str:\n return self.get_filter_str()\n" }, { "path": "src/palimpzest/core/elements/groupbysig.py", "content": "from __future__ import annotations\n\nfrom typing import Any\n\nfrom pydantic import BaseModel\n\nfrom palimpzest.core.lib.schemas import create_schema_from_fields\n\n# TODO:\n# - move the arguments for group_by_fields, agg_funcs, and agg_fields into the Dataset.groupby() operator\n# - construct the correct output schema using the input schema and the group by and aggregation fields\n# - remove/update all other references to GroupBySig in the codebase\n\n# TODO:\n# - move the arguments for group_by_fields, agg_funcs, and agg_fields into the Dataset.groupby() operator\n# - construct the correct output schema using the input schema and the group by and aggregation fields\n# - remove/update all other references to GroupBySig in the codebase\n\n# signature for a group by aggregate that applies\n# group and aggregation to an input tuple\nclass GroupBySig:\n def __init__(self, group_by_fields: list[str], agg_funcs: list[str], agg_fields: list[str]):\n self.group_by_fields = group_by_fields\n self.agg_funcs = agg_funcs\n self.agg_fields = agg_fields\n\n def validate_schema(self, input_schema: type[BaseModel]) -> tuple[bool, str | None]:\n for f in self.group_by_fields:\n if f not in input_schema.model_fields:\n return (False, \"Supplied schema has no field \" + f)\n for f in self.agg_fields:\n if f not in input_schema.model_fields:\n return (False, \"Supplied schema has no field \" + f)\n return (True, None)\n\n def serialize(self) -> dict[str, Any]:\n out = {\n \"group_by_fields\": self.group_by_fields,\n \"agg_funcs\": self.agg_funcs,\n \"agg_fields\": self.agg_fields,\n }\n return out\n\n def __str__(self) -> str:\n return \"GroupBy(\" + repr(self.serialize()) + \")\"\n\n def __hash__(self) -> int:\n # custom hash function\n return hash(repr(self.serialize()))\n\n def __eq__(self, other) -> bool:\n # __eq__ should be defined for consistency with __hash__\n return isinstance(other, GroupBySig) and self.serialize() == other.serialize()\n\n def get_agg_field_names(self) -> list[str]:\n ops = []\n for i in range(0, len(self.agg_fields)):\n ops.append(self.agg_funcs[i] + \"(\" + self.agg_fields[i] + \")\")\n return ops\n\n # TODO: output schema needs to account for input schema types and create new output schema types\n def output_schema(self) -> type[BaseModel]:\n # the output class varies depending on the group by, so here\n # we dynamically construct this output\n fields = []\n for g in self.group_by_fields:\n f = {\"name\": g, \"type\": Any, \"desc\": f\"Group by field: {g}\"}\n fields.append(f)\n\n ops = self.get_agg_field_names()\n for op in ops:\n f = {\"name\": op, \"type\": Any, \"desc\": f\"Aggregate field: {op}\"}\n fields.append(f)\n\n return create_schema_from_fields(fields)\n" }, { "path": "src/palimpzest/core/elements/records.py", "content": "from __future__ import annotations\n\nimport json\nfrom collections.abc import Generator\nfrom copy import deepcopy\nfrom typing import Any\n\nimport pandas as pd\nfrom pydantic import BaseModel\nfrom pydantic.fields import FieldInfo\n\nfrom palimpzest.core.data import context\nfrom palimpzest.core.lib.schemas import (\n AUDIO_FIELD_TYPES,\n IMAGE_FIELD_TYPES,\n AudioBase64,\n AudioFilepath,\n ImageBase64,\n ImageFilepath,\n ImageURL,\n project,\n union_schemas,\n)\nfrom palimpzest.core.models import ExecutionStats, PlanStats, RecordOpStats\nfrom palimpzest.utils.hash_helpers import hash_for_id\n\n\nclass DataRecord:\n \"\"\"A DataRecord is a single record of data matching some schema defined by a BaseModel.\"\"\"\n\n def __init__(\n self,\n data_item: BaseModel,\n source_indices: str | int | list[str | int],\n parent_ids: str | list[str] | None = None,\n cardinality_idx: int | None = None,\n ):\n # check that source_indices are provided\n assert source_indices is not None, \"Every DataRecord must be constructed with source index (or indices)\"\n\n # normalize to list[str]\n if not isinstance(source_indices, list):\n source_indices = [source_indices]\n\n # normalize to list[str]\n if isinstance(parent_ids, str):\n parent_ids = [parent_ids]\n\n # data for the data record\n self._data_item = data_item\n\n # the index in the root Dataset from which this DataRecord is derived;\n # each source index takes the form: f\"{root_dataset.id}-{idx}\"\n self._source_indices = sorted(source_indices)\n\n # the id(s) of the parent record(s) from which this DataRecord is derived\n self._parent_ids = parent_ids\n\n # store the cardinality index\n self._cardinality_idx = cardinality_idx\n\n # indicator variable which may be flipped by filter operations to signal when a record has been filtered out\n self._passed_operator = True\n\n # NOTE: Record ids are hashed based on:\n # 0. their schema (keys)\n # 1. their parent record id(s) (or source_indices if there is no parent record)\n # 2. their index in the fan out (if this is in a one-to-many operation)\n #\n # We currently do NOT hash just based on record content (i.e. schema (key, value) pairs)\n # because multiple outputs for a given operation may have the exact same\n # schema (key, value) pairs.\n #\n # We may revisit this hashing scheme in the future.\n\n # unique identifier for the record\n schema_fields = sorted(list(type(data_item).model_fields))\n id_str = (\n str(schema_fields) + str(parent_ids) if parent_ids is not None else str(self._source_indices)\n if cardinality_idx is None\n else str(schema_fields) + str(cardinality_idx) + str(parent_ids) if parent_ids is not None else str(self._source_indices)\n )\n self._id = hash_for_id(id_str)\n\n\n # TODO: raise an exception if one of these fields is present in the schema\n # - put these in a constant list up top\n # - import the constant list in Dataset (if possible) and check at plan creation time\n def __setattr__(self, name: str, value: Any, /) -> None:\n if name in [\"_data_item\", \"_source_indices\", \"_parent_ids\", \"_cardinality_idx\", \"_passed_operator\", \"_id\"]:\n super().__setattr__(name, value)\n else:\n setattr(self._data_item, name, value)\n\n\n def __getattr__(self, name: str) -> Any:\n return getattr(self._data_item, name)\n\n\n def __getitem__(self, field: str) -> Any:\n return getattr(self._data_item, field)\n\n\n def __setitem__(self, field: str, value: Any) -> None:\n setattr(self._data_item, field, value)\n\n\n def __str__(self, truncate: int | None = 15) -> str:\n if truncate is not None:\n items = (f\"{k}={str(v)[:truncate]!r}{'...' if len(str(v)) > truncate else ''}\" for k, v in sorted(self._data_item.model_dump().items()))\n else:\n items = (f\"{k}={v!r}\" for k, v in sorted(self._data_item.model_dump().items()))\n return \"{}({})\".format(type(self).__name__, \", \".join(items))\n\n\n def __repr__(self) -> str:\n return self.__str__(truncate=None)\n\n\n def __eq__(self, other):\n return isinstance(other, DataRecord) and self._data_item == other._data_item\n\n\n def __hash__(self):\n return hash(self.to_json_str(bytes_to_str=True, sorted=True))\n\n\n def __iter__(self):\n yield from self._data_item.__iter__()\n\n\n def get_field_names(self):\n return list(type(self._data_item).model_fields.keys())\n\n\n def get_field_type(self, field_name: str) -> FieldInfo:\n return type(self._data_item).model_fields[field_name]\n\n @property\n def schema(self) -> type[BaseModel]:\n return type(self._data_item)\n\n def copy(self) -> DataRecord:\n # get the set of fields to copy from the parent record\n copy_field_names = [field.split(\".\")[-1] for field in self.get_field_names()]\n\n # copy field types and values from the parent\n data_item = {field_name: self[field_name] for field_name in copy_field_names}\n\n # make copy of the current record\n new_dr = DataRecord(\n self.schema(**data_item),\n source_indices=self._source_indices,\n parent_ids=self._parent_ids,\n cardinality_idx=self._cardinality_idx,\n )\n\n # copy the passed_operator attribute\n new_dr._passed_operator = self._passed_operator\n\n return new_dr\n\n @staticmethod\n def from_parent(\n schema: type[BaseModel],\n data_item: dict,\n parent_record: DataRecord,\n project_cols: list[str] | None = None,\n cardinality_idx: int | None = None,\n ) -> DataRecord:\n # if project_cols is None, then the new schema is a union of the provided schema and parent_record.schema;\n # if project_cols is an empty list, then the new schema is simply the provided schema\n # otherwise, it's a ProjectSchema\n new_schema = None\n if project_cols is None:\n new_schema = union_schemas([schema, parent_record.schema])\n elif project_cols == []:\n new_schema = schema\n else:\n new_schema = union_schemas([schema, parent_record.schema])\n new_schema = project(new_schema, project_cols)\n\n # get the set of fields and field descriptions to copy from the parent record\n copy_field_names = parent_record.get_field_names() if project_cols is None else project_cols\n copy_field_names = [field.split(\".\")[-1] for field in copy_field_names]\n\n # copy fields from the parent\n data_item.update({field_name: parent_record[field_name] for field_name in copy_field_names})\n\n # corner-case: wrap values in lists if the new schema expects a list but the data item has a single value\n for field_name, field_info in new_schema.model_fields.items():\n field_should_be_list = hasattr(field_info.annotation, '__origin__') and field_info.annotation.__origin__ is list\n field_is_not_list = field_name in data_item and not isinstance(data_item[field_name], list)\n if field_should_be_list and field_is_not_list:\n data_item[field_name] = [data_item[field_name]]\n\n # make new record which has parent_record as its parent (and the same source_indices)\n new_dr = DataRecord(\n new_schema(**data_item),\n source_indices=parent_record._source_indices,\n parent_ids=[parent_record._id],\n cardinality_idx=cardinality_idx,\n )\n\n return new_dr\n\n @staticmethod\n def from_agg_parents(\n data_item: BaseModel,\n parent_records: DataRecordSet,\n cardinality_idx: int | None = None,\n ) -> DataRecord:\n # flatten source indices from all parents\n source_indices = [\n source_idx\n for parent_record in parent_records\n for source_idx in parent_record._source_indices\n ]\n\n # make new record which has all parent records as its parents\n return DataRecord(\n data_item,\n source_indices=source_indices,\n parent_ids=[parent_record._id for parent_record in parent_records],\n cardinality_idx=cardinality_idx,\n )\n\n @staticmethod\n def from_join_parents(\n schema: type[BaseModel],\n left_parent_record: DataRecord | None,\n right_parent_record: DataRecord | None,\n project_cols: list[str] | None = None,\n cardinality_idx: int = None,\n ) -> DataRecord:\n # get the set of fields and field descriptions to copy from the parent record(s)\n left_copy_field_names = [] if left_parent_record is None else (\n left_parent_record.get_field_names()\n if project_cols is None\n else [col for col in project_cols if col in left_parent_record.get_field_names()]\n )\n right_copy_field_names = [] if right_parent_record is None else (\n right_parent_record.get_field_names()\n if project_cols is None\n else [col for col in project_cols if col in right_parent_record.get_field_names()]\n )\n left_copy_field_names = [field.split(\".\")[-1] for field in left_copy_field_names]\n right_copy_field_names = [field.split(\".\")[-1] for field in right_copy_field_names]\n\n # copy fields from the parents\n data_item = {field_name: left_parent_record[field_name] for field_name in left_copy_field_names}\n for field_name in right_copy_field_names:\n new_field_name = field_name\n if field_name in left_copy_field_names:\n new_field_name = f\"{field_name}_right\"\n data_item[new_field_name] = right_parent_record[field_name]\n\n # for any missing fields in the schema, set them to None\n for field_name in schema.model_fields:\n if field_name not in data_item:\n data_item[field_name] = None\n\n # make new record which has left and right parent record as its parents\n left_parent_source_indices = [] if left_parent_record is None else list(left_parent_record._source_indices)\n right_parent_source_indices = [] if right_parent_record is None else list(right_parent_record._source_indices)\n left_parent_record_id = [] if left_parent_record is None else [left_parent_record._id]\n right_parent_record_id = [] if right_parent_record is None else [right_parent_record._id]\n new_dr = DataRecord(\n schema(**data_item),\n source_indices=left_parent_source_indices + right_parent_source_indices,\n parent_ids=left_parent_record_id + right_parent_record_id,\n cardinality_idx=cardinality_idx,\n )\n\n return new_dr\n\n @staticmethod\n def to_df(records: list[DataRecord], project_cols: list[str] | None = None) -> pd.DataFrame:\n if len(records) == 0:\n return pd.DataFrame()\n\n fields = records[0].get_field_names()\n if project_cols is not None and len(project_cols) > 0:\n fields = [field for field in fields if field in project_cols]\n\n # convert Context --> str\n for record in records:\n for k in fields:\n if isinstance(record[k], context.Context):\n record[k] = record[k].description\n\n return pd.DataFrame([\n {k: record[k] for k in fields}\n for record in records\n ])\n\n def to_json_str(self, include_bytes: bool = True, bytes_to_str: bool = False, project_cols: list[str] | None = None, sorted: bool = False):\n \"\"\"Return a JSON representation of this DataRecord\"\"\"\n record_dict = self.to_dict(include_bytes, bytes_to_str, project_cols, sorted)\n return json.dumps(record_dict, indent=2)\n\n def to_dict(self, include_bytes: bool = True, bytes_to_str: bool = False, project_cols: list[str] | None = None, _sorted: bool = False, mask_filepaths: bool = False):\n \"\"\"Return a dictionary representation of this DataRecord\"\"\"\n # TODO(chjun): In case of numpy types, the json.dumps will fail. Convert to native types.\n # Better ways to handle this.\n field_values = {\n k: v.description if isinstance(v, context.Context) else v\n for k, v in self._data_item.model_dump().items()\n }\n dct = pd.Series(field_values).to_dict()\n\n if project_cols is not None and len(project_cols) > 0:\n project_field_names = set(field.split(\".\")[-1] for field in project_cols)\n dct = {k: v for k, v in dct.items() if k in project_field_names}\n\n if not include_bytes:\n bytes_field_types = [bytes, list[bytes], bytes | None, list[bytes] | None, bytes | Any, list[bytes] | Any]\n bytes_field_types += AUDIO_FIELD_TYPES + IMAGE_FIELD_TYPES\n for k in dct:\n field_type = self.get_field_type(k)\n if field_type.annotation in bytes_field_types:\n dct[k] = \"\"\n\n if bytes_to_str:\n for k, v in dct.items():\n if isinstance(v, bytes):\n dct[k] = v.decode(\"utf-8\")\n elif isinstance(v, list) and len(v) > 0 and any([isinstance(elt, bytes) for elt in v]):\n dct[k] = [elt.decode(\"utf-8\") if isinstance(elt, bytes) else elt for elt in v]\n\n if _sorted:\n dct = dict(sorted(dct.items()))\n\n if mask_filepaths:\n for k in dct:\n field_type = self.get_field_type(k)\n if field_type.annotation in [AudioBase64, AudioFilepath, ImageBase64, ImageFilepath, ImageURL]:\n dct[k] = \"\"\n\n return deepcopy(dct)\n\n\nclass DataRecordSet:\n \"\"\"\n A DataRecordSet contains a list of DataRecords that share the same schema, same parent(s), and same source(s).\n\n We explicitly check that this is True.\n\n The record_op_stats could be empty if the DataRecordSet is not from executing an operator.\n \"\"\"\n def __init__(\n self,\n data_records: list[DataRecord],\n record_op_stats: list[RecordOpStats],\n field_to_score_fn: dict[str, str | callable] | None = None,\n input: int | DataRecord | list[DataRecord] | tuple[list[DataRecord]] | None = None,\n ):\n # set data_records, parent_ids, and source_indices; note that it is possible for\n # data_records to be an empty list in the event of a failed convert\n self.data_records = data_records\n self.parent_ids = data_records[0]._parent_ids if len(data_records) > 0 else None\n self.source_indices = data_records[0]._source_indices if len(data_records) > 0 else None\n self.schema = data_records[0].schema if len(data_records) > 0 else None\n\n # the input to the operator which produced the data_records; type is tuple[DataRecord] | tuple[int]\n # - for scan operators, input is a singleton tuple[int] which wraps the source_idx, e.g.: (source_idx,)\n # - for join operators, input is a tuple with one entry for the left input DataRecord and one entry for the right input DataRecord\n # - for aggregate operators, input is a tuple with all the input DataRecords to the aggregation\n # - for all other operaotrs, input is a singleton tuple[DataRecord] which wraps the single input\n self.input = input\n\n # set statistics for generating these records\n self.record_op_stats = record_op_stats\n\n # assign field_to_score_fn if provided\n self.field_to_score_fn = {} if field_to_score_fn is None else field_to_score_fn\n\n def get_total_cost(self) -> float:\n return sum([record_op_stats.cost_per_record for record_op_stats in self.record_op_stats])\n\n def get_field_to_score_fn(self) -> dict[str, str | callable]:\n return self.field_to_score_fn\n\n def __getitem__(self, slice) -> DataRecord | list[DataRecord]:\n return self.data_records[slice]\n\n def __len__(self) -> int:\n return len(self.data_records)\n\n def __iter__(self) -> Generator[DataRecord]:\n yield from self.data_records\n\n\nclass DataRecordCollection:\n \"\"\"\n A DataRecordCollection contains a list of DataRecords.\n\n This is a wrapper class for list[DataRecord] to support more advanced features for output of execute().\n\n The difference between DataRecordSet and DataRecordCollection \n\n Goal: \n DataRecordSet is a set of DataRecords that share the same schema, same parents, and same sources.\n DataRecordCollection is a general wrapper for list[DataRecord].\n \n Usage:\n DataRecordSet is used for the output of executing an operator.\n DataRecordCollection is used for the output of executing a query, we definitely could extend it to support more advanced features for output of execute().\n \"\"\"\n def __init__(self, data_records: list[DataRecord], execution_stats: ExecutionStats | None = None, plan_stats: PlanStats | None = None):\n self.data_records = data_records\n self.execution_stats = execution_stats\n self.plan_stats = plan_stats\n self.executed_plans = self._get_executed_plans()\n\n def __iter__(self) -> Generator[DataRecord]:\n \"\"\"Allow iterating directly over the data records\"\"\"\n yield from self.data_records\n\n def __len__(self):\n \"\"\"Return the number of records in the collection\"\"\"\n return len(self.data_records)\n\n def to_df(self, cols: list[str] | None = None):\n return DataRecord.to_df(self.data_records, cols)\n\n def _get_executed_plans(self):\n if self.plan_stats is not None:\n return [self.plan_stats.plan_str]\n elif self.execution_stats is not None:\n return list(self.execution_stats.plan_strs.values())\n else:\n return None\n" }, { "path": "src/palimpzest/core/lib/__init__.py", "content": "" }, { "path": "src/palimpzest/core/lib/schemas.py", "content": "from __future__ import annotations\n\nimport sys\nfrom typing import Any, TypeAliasType\n\nimport pandas as pd\nfrom pydantic import BaseModel, Field, create_model\nfrom pydantic.fields import FieldInfo\n\nfrom palimpzest.utils.hash_helpers import hash_for_serialized_dict\n\n# DEFINITIONS\nPANDAS_DTYPE_TO_PYDANTIC = {\n \"object\": str,\n \"bool\": bool,\n \"int64\": int,\n \"float64\": float,\n}\n\n# IMAGE TYPES\nImageFilepath = TypeAliasType('ImageFilepath', str)\nImageBase64 = TypeAliasType('ImageBase64', str)\nImageURL = TypeAliasType('ImageURL', str)\n\n# AUDIO TYPES\nAudioFilepath = TypeAliasType('AudioFilepath', str)\nAudioBase64 = TypeAliasType('AudioBase64', str)\n\nIMAGE_LIST_FIELD_TYPES = [\n list[ImageBase64],\n list[ImageFilepath],\n list[ImageURL],\n list[ImageBase64] | None,\n list[ImageFilepath] | None,\n list[ImageURL] | None,\n list[ImageBase64] | Any,\n list[ImageFilepath] | Any,\n list[ImageURL] | Any,\n]\nIMAGE_FIELD_TYPES = IMAGE_LIST_FIELD_TYPES + [\n ImageBase64, ImageFilepath, ImageURL,\n ImageBase64 | None, ImageFilepath | None, ImageURL | None,\n ImageBase64 | Any, ImageFilepath | Any, ImageURL | Any,\n]\nAUDIO_LIST_FIELD_TYPES = [\n list[AudioBase64],\n list[AudioFilepath],\n list[AudioBase64] | None,\n list[AudioFilepath] | None,\n list[AudioBase64] | Any,\n list[AudioFilepath] | Any,\n]\nAUDIO_FIELD_TYPES = AUDIO_LIST_FIELD_TYPES + [\n AudioBase64, AudioFilepath,\n AudioBase64 | None, AudioFilepath | None,\n AudioBase64 | Any, AudioFilepath | Any,\n]\n\n\ndef get_schema_field_names(schema: type[BaseModel], id: str | None = None) -> list[str]:\n \"\"\"Return the field names of a Pydantic model.\"\"\"\n return list(schema.model_fields) if id is None else [f\"{schema.__name__}.{id}.{field_name}\" for field_name in schema.model_fields]\n\n\ndef _create_pickleable_model(fields: dict[str, tuple[type, FieldInfo]]) -> type[BaseModel]:\n \"\"\"Create a Pydantic model that can be pickled.\"\"\"\n # create unique name for the unioned model\n new_schema_name = f\"Schema{sorted(fields.keys())}\"\n new_schema_id = hash_for_serialized_dict({\n field_name: {\"annotation\": str(annotation), \"default\": str(field.default), \"description\": field.description}\n for field_name, (annotation, field) in fields.items()\n })\n\n # if this class already exists, get it from the module and return\n module = sys.modules[__name__]\n if hasattr(module, new_schema_id):\n return getattr(module, new_schema_id)\n\n # create the class dynamically\n new_model = create_model(new_schema_name, **fields)\n\n # register it in the module's namespace so pickle can find it\n module = sys.modules[__name__]\n setattr(module, new_schema_id, new_model)\n new_model.__module__ = module.__name__\n\n return new_model\n\n\ndef relax_schema(model: type[BaseModel]) -> type[BaseModel]:\n \"\"\"Updates the type annotation for every field in the BaseModel to include typing.Any\"\"\"\n fields = {}\n for field_name, field in model.model_fields.items():\n fields[field_name] = (field.annotation | Any, field)\n\n return _create_pickleable_model(fields)\n\n\ndef project(model: type[BaseModel], project_fields: list[str]) -> type[BaseModel]:\n \"\"\"Project a Pydantic model to only the specified columns.\"\"\"\n # make sure projection column names are shortened\n project_fields = [field_name.split(\".\")[-1] for field_name in project_fields]\n\n # build up the fields for the new schema\n fields = {}\n for field_name, field in model.model_fields.items():\n if field_name in project_fields:\n fields[field_name] = (field.annotation, field)\n\n # create and return the new schema\n return _create_pickleable_model(fields)\n\n\ndef create_schema_from_fields(fields: list[dict]) -> type[BaseModel]:\n \"\"\"Create a Pydantic model from a list of fields.\"\"\"\n fields_ = {}\n for field in fields:\n assert \"name\" in field, \"fields must contain a 'name' key\"\n assert \"type\" in field, \"fields must contain a 'type' key\"\n assert \"desc\" in field or \"description\" in field, \"fields must contain a 'description' key\"\n\n # for backwards compatability, rename \"desc\" to \"description\"\n if \"desc\" in field:\n field[\"description\"] = field.pop(\"desc\")\n field_name = field[\"name\"]\n field_type = field[\"type\"]\n fields_[field_name] = (field_type, Field(**{k: v for k, v in field.items() if k not in [\"name\", \"type\"]}))\n\n return _create_pickleable_model(fields_)\n\n\ndef create_schema_from_df(df: pd.DataFrame) -> type[BaseModel]:\n \"\"\"Create a Pydantic model from a Pandas DataFrame.\"\"\"\n fields = {}\n for column, dtype in zip(df.columns, df.dtypes):\n column = f\"column_{column}\" if isinstance(column, int) else column\n field_desc = f\"The {column} column from an input DataFrame\"\n annotation = PANDAS_DTYPE_TO_PYDANTIC.get(str(dtype), Any)\n fields[column] = (annotation, Field(description=field_desc))\n\n # create and return the new schema\n return _create_pickleable_model(fields)\n\n\ndef union_schemas(models: list[type[BaseModel]], join: bool = False, on: list[str] | None = None) -> type[BaseModel]:\n \"\"\"Union multiple Pydantic models into a single model.\"\"\"\n # convert on to empty list if None\n if on is None:\n on = []\n\n # build up the fields for the new schema\n fields = {}\n for model in models:\n for field_name, field in model.model_fields.items():\n # for non-join unions, make sure duplicate fields have the same type\n if not join and field_name in fields:\n assert fields[field_name][0] == field.annotation, f\"Field {field_name} has different types in different models\"\n\n # for joins with \"on\" specified, no need to rename fields in \"on\"\n elif join and field_name in on and field_name in fields:\n continue\n\n # otherwise, rename duplicate fields by appending _right\n elif join and field_name in fields:\n while field_name in fields:\n field_name = f\"{field_name}_right\"\n\n # add the field to the new schema\n fields[field_name] = (field.annotation, field)\n\n # create and return the new schema\n return _create_pickleable_model(fields)\n\n###################################################################################\n# \"Core\" useful Schemas. These are Schemas that almost everyone will need.\n# File, TextFile, Image, PDF, etc.\n###################################################################################\n\n\n# First-level Schema's\nclass DefaultSchema(BaseModel):\n \"\"\"Store context data.\"\"\"\n value: Any = Field(description=\"The value of the input data\")\n\nclass Download(BaseModel):\n \"\"\"A download is a URL and the contents of the download.\"\"\"\n url: str = Field(description=\"The URL of the download\")\n content: bytes = Field(description=\"The contents of the download\")\n timestamp: str = Field(description=\"The timestamp of the download\")\n\nclass File(BaseModel):\n \"\"\"\n A File is defined by two Fields:\n - the filename (string)\n - the contents of the file (bytes)\n \"\"\"\n filename: str = Field(description=\"The UNIX-style name of the file\")\n contents: bytes = Field(description=\"The contents of the file\")\n\nclass TextFile(BaseModel):\n \"\"\"A text file is a File that contains only text. No binary data.\"\"\"\n filename: str = Field(description=\"The UNIX-style name of the file\")\n contents: str = Field(description=\"The contents of the file\")\n\nclass Average(BaseModel):\n average: float = Field(description=\"The average value of items in the dataset\")\n\nclass Count(BaseModel):\n count: int = Field(description=\"The count of items in the dataset\")\n\nclass Sum(BaseModel):\n sum: int = Field(description=\"The summation of items in the dataset\")\n\nclass Min(BaseModel):\n min: int | float = Field(description=\"The minimum value of some items in the dataset\")\n\nclass Max(BaseModel):\n max: int | float = Field(description=\"The maximum value of some items in the dataset\")\n\nclass OperatorDerivedSchema(BaseModel):\n \"\"\"Schema defined by an operator, e.g., a join or a group by\"\"\"\n\nclass Table(BaseModel):\n \"\"\"A Table is an object composed of a header and rows.\"\"\"\n filename: str = Field(description=\"The name of the file the table was extracted from\")\n name: str = Field(description=\"The name of the table\")\n header: list[str] = Field(description=\"The header of the table\")\n rows: list[list] = Field(description=\"The rows of the table\")\n\nclass URL(BaseModel):\n \"\"\"A URL is a string that represents a web address.\"\"\"\n url: str = Field(description=\"A URL\")\n\nclass WebPage(BaseModel):\n \"\"\"A web page is a URL and the contents of the page.\"\"\"\n text: str = Field(description=\"The text contents of the web page\")\n html: str = Field(description=\"The html contents of the web page\")\n timestamp: str = Field(description=\"The timestamp of the download\")\n filename: str = Field(description=\"The name of the file the web page was downloaded from\")\n\n# Second-level Schemas\nclass ImageFile(File):\n \"\"\"A file that contains an image.\"\"\"\n contents: ImageBase64 = Field(description=\"The contents of the image encoded as a base64 string\")\n\nclass AudioFile(File):\n \"\"\"A file that contains audio.\"\"\"\n contents: AudioBase64 = Field(description=\"The contents of an audio recording encoded as a base64 string\")\n\nclass PDFFile(File):\n \"\"\"A PDF file is a File that is a PDF. It has specialized fields, font information, etc.\"\"\"\n # This class is currently very impoverished. It needs a lot more fields before it can correctly represent a PDF.\n text_contents: str = Field(description=\"The text-only contents of the PDF\")\n\nclass XLSFile(File):\n \"\"\"An XLS file is a File that contains one or more Excel spreadsheets.\"\"\"\n number_sheets: int = Field(description=\"The number of sheets in the Excel file\")\n sheet_names: list[str] = Field(description=\"The names of the sheets in the Excel file\")\n\n# Third-level Schemas\nclass EquationImage(ImageFile):\n \"\"\"An image that contains a mathematical equation.\"\"\"\n equation_text: str = Field(description=\"The text representation of the equation in the image\")\n\nclass PlotImage(ImageFile):\n \"\"\"An image that contains a plot, such as a graph or chart.\"\"\"\n plot_description: str = Field(description=\"A description of the plot\")\n" }, { "path": "src/palimpzest/core/models.py", "content": "from __future__ import annotations\n\nimport json\nimport time\nfrom abc import abstractmethod\nfrom typing import Any\n\nfrom pydantic import BaseModel, Field\n\n\nclass GenerationStats(BaseModel):\n \"\"\"\n Model for storing statistics about the execution of an operator on a single record.\n \"\"\"\n\n model_name: str | None = None\n\n # The raw answer as output from the generator (a list of strings, possibly of len 1)\n # raw_answers: Optional[List[str]] = field(default_factory=list)\n\n # the total number of input text tokens processed by this operator; None if this operation did not use any LLM\n # typed as a float because GenerationStats may be amortized (i.e. divided) acorss a number of output records\n input_text_tokens: float = 0.0\n\n # the total number of input audio tokens processed by this operation.\n input_audio_tokens: float = 0.0\n\n # the total number of input image tokens processed by this operation.\n input_image_tokens: float = 0.0\n\n # the total number of cache read tokens processed by this operation (charged at a discount, typically 0.1x input rate)\n cache_read_tokens: float = 0.0\n\n # the total number of tokens written to the cache in this operation (Anthropic only) (charged at creation rate, typically 1.25x input rate)\n cache_creation_tokens: float = 0.0\n\n # the number of output text tokens generated by the model\n output_text_tokens: float = 0.0\n\n # the total number of input tokens processed by embedding models\n embedding_input_tokens: float = 0.0\n\n # the total cost of processing the input and output tokens; None if this operation did not use an LLM\n # TODO: future PR: cost_per_record --> total_cost\n cost_per_record: float = 0.0\n\n # (if applicable) the time (in seconds) spent executing a call to an LLM\n llm_call_duration_secs: float = 0.0\n\n # (if applicable) the time (in seconds) spent executing a call to a function\n fn_call_duration_secs: float = 0.0\n\n # (if applicable) the total number of LLM calls made by this operator\n total_llm_calls: float = 0.0\n\n # (if applicable) the total number of embedding LLM calls made by this operator\n total_embedding_llm_calls: float = 0.0\n\n def __iadd__(self, other: GenerationStats) -> GenerationStats:\n for field in type(self).model_fields:\n if field == \"model_name\":\n continue\n setattr(self, field, getattr(self, field) + getattr(other, field))\n return self\n\n def __add__(self, other: GenerationStats) -> GenerationStats:\n dct = {\n field: getattr(self, field) + getattr(other, field)\n for field in type(self).model_fields\n if field != \"model_name\"\n }\n dct[\"model_name\"] = self.model_name\n return GenerationStats(**dct)\n\n # Do the same as iadd and add but with division operator\n def __itruediv__(self, quotient: float) -> GenerationStats:\n if quotient == 0:\n raise ZeroDivisionError(\"Cannot divide by zero\")\n if isinstance(quotient, int):\n quotient = float(quotient)\n for field in type(self).model_fields:\n if field == \"model_name\":\n continue\n setattr(self, field, getattr(self, field) / quotient)\n return self\n\n def __truediv__(self, quotient: float) -> GenerationStats:\n if quotient == 0:\n raise ZeroDivisionError(\"Cannot divide by zero\")\n if isinstance(quotient, int):\n quotient = float(quotient)\n dct = {\n field: getattr(self, field) / quotient\n for field in type(self).model_fields\n if field != \"model_name\"\n }\n dct[\"model_name\"] = self.model_name\n return GenerationStats(**dct)\n\n def __radd__(self, other: int) -> GenerationStats:\n assert not isinstance(other, GenerationStats), \"This should not be called with a GenerationStats object\"\n return self\n\n # NOTE: this is added temporarily to help track cost of compute agent writing PZ code;\n # once we find a long-term solution for tracking that cost, we can remove this\n def to_json(self, filepath: str | None = None) -> dict | None:\n if filepath is None:\n return self.model_dump(mode=\"json\")\n\n with open(filepath, \"w\") as f:\n json.dump(self.model_dump(mode=\"json\"), f)\n\n\nclass RecordOpStats(BaseModel):\n \"\"\"\n Model for storing statistics about the execution of an operator on a single record.\n \"\"\"\n\n ##### REQUIRED FIELDS #####\n # record id; an identifier for this record\n record_id: str | int\n\n # identifier for the parent(s) of this record\n record_parent_ids: list[str | int] | None\n\n # idenifier for the source indices of this record\n record_source_indices: list[str | int]\n\n # a dictionary with the record state after being processed by the operator\n record_state: dict[str, Any]\n\n # operation id; an identifier for this operation's physical op id\n full_op_id: str\n\n # logical operation id; the logical op id for this physical op\n logical_op_id: str\n\n # operation name\n op_name: str\n\n # the time spent by the data record just in this operation\n time_per_record: float\n\n # the cost (in dollars) to generate this record at this operation\n cost_per_record: float\n\n ##### NOT-OPTIONAL, BUT FILLED BY EXECUTION CLASS AFTER CONSTRUCTOR CALL #####\n # the ID(s) of the physical operation(s) which produced the input record(s) for this record at this operation\n source_unique_full_op_ids: list[str] | None = None\n\n # the ID(s) of the logical operation(s) which produced the input record(s) for this record at this operation\n source_unique_logical_op_ids: list[str] | None = None\n\n # the ID of the physical plan which produced this record at this operation\n plan_id: str = \"\"\n\n ##### OPTIONAL, BUT FILLED BY COST MODEL AFTER SAMPLE DATA EXECUTION #####\n quality: float | None = None\n\n ##### OPTIONAL FIELDS (I.E. ONLY MANDATORY FOR CERTAIN OPERATORS) #####\n # (if applicable) the name of the model used to generate the output for this record\n model_name: str | None = None\n\n # (if applicable) the mapping from field-name to generated output for this record\n answer: dict[str, Any] | None = None\n\n # (if applicable) the mapping from field-name to generated output for this record\n # raw_answers: Optional[List[str, Any]] = field(default_factory=list)\n\n # (if applicable) the list of input fields for the generation for this record\n input_fields: list[str] | None = None\n\n # (if applicable) the list of generated fields for this record\n generated_fields: list[str] | None = None\n\n # the number of input text tokens processed by this operation\n # typed as a float because GenerationStats may be amortized (i.e. divided) across a number of output records\n input_text_tokens: float = 0.0\n\n # the number of input audio tokens processed by this operation\n input_audio_tokens: float = 0.0\n\n # the number of input image tokens processed by this operation\n input_image_tokens: float = 0.0\n\n # the number of cache read tokens processed by this operation\n cache_read_tokens: float = 0.0\n\n # the number of tokens written to cache by this operation\n cache_creation_tokens: float = 0.0\n\n # the number of output text tokens generated by this operation\n output_text_tokens: float = 0.0\n\n # the number of input tokens processed by embedding models\n embedding_input_tokens: float = 0.0\n\n # (if applicable) the filter text (or a string representation of the filter function) applied to this record\n filter_str: str | None = None\n\n # (if applicable) the join condition applied to this record\n join_condition: str | None = None\n\n # the True/False result of whether this record was output by the operator or not\n # (can only be False if the operator is a Filter or Join)\n passed_operator: bool = True\n\n # (if applicable) the time (in seconds) spent executing a call to an LLM\n llm_call_duration_secs: float = 0.0\n\n # (if applicable) the time (in seconds) spent executing a UDF or calling an external api\n fn_call_duration_secs: float = 0.0\n\n # (if applicable) the total number of LLM calls made by this operator\n total_llm_calls: float = 0.0\n\n # (if applicable) the total number of embedding LLM calls made by this operator\n total_embedding_llm_calls: float = 0.0\n\n # (if applicable) a boolean indicating whether this is the statistics captured from a failed convert operation\n failed_convert: bool | None = None\n\n # an OPTIONAL dictionary with more detailed information about this operation;\n op_details: dict[str, Any] = Field(default_factory=dict)\n\n\nclass OperatorStats(BaseModel):\n \"\"\"\n Model for storing statistics captured within a given operator.\n \"\"\"\n\n # the full ID of the physical operation in which these stats were collected\n full_op_id: str\n\n # the name of the physical operation in which these stats were collected\n op_name: str\n\n # the total time spent in this operation\n total_op_time: float = 0.0\n\n # the total cost of this operation\n total_op_cost: float = 0.0\n\n # the number of input text tokens processed by this operation\n input_text_tokens: float = 0.0\n\n # the number of input audio tokens processed by this operation\n input_audio_tokens: float = 0.0\n\n # the number of input image tokens processed by this operation\n input_image_tokens: float = 0.0\n\n # the number of cache read tokens processed by this operation\n cache_read_tokens: float = 0.0\n\n # the number of tokens written to cache by this operation\n cache_creation_tokens: float = 0.0\n\n # the number of output text tokens generated by this operation\n output_text_tokens: float = 0.0\n\n # the number of input tokens processed by embedding models\n embedding_input_tokens: float = 0.0\n\n # a list of RecordOpStats processed by the operation\n record_op_stats_lst: list[RecordOpStats] = Field(default_factory=list)\n\n # the unique full ID(s) of the physical operator(s) which precede this one (used by PlanStats)\n source_unique_full_op_ids: list[str] | None = None\n\n # the unique full ID(s) of the logical operator(s) which precede this one (used by SentinelPlanStats)\n source_unique_logical_op_ids: list[str] | None = None\n\n # the ID of the physical plan which this operator is part of\n plan_id: str = \"\"\n\n # an OPTIONAL dictionary with more detailed information about this operation;\n op_details: dict[str, Any] = Field(default_factory=dict)\n\n def __iadd__(self, stats: OperatorStats | RecordOpStats) -> OperatorStats:\n \"\"\"\n Sum the given stats to this operator's stats. The given stats can be either:\n\n 1. an OperatorStats object\n 2. a RecordOpStats object\n\n NOTE: in case (1.) we assume the execution layer guarantees that `stats` is\n generated by the same operator in the same plan. Thus, we assume the\n full_op_ids, op_name, source_op_id, etc. do not need to be updated.\n \"\"\"\n if isinstance(stats, OperatorStats):\n self.total_op_time += stats.total_op_time\n self.total_op_cost += stats.total_op_cost\n self.input_text_tokens += stats.input_text_tokens\n self.input_audio_tokens += stats.input_audio_tokens\n self.input_image_tokens += stats.input_image_tokens\n self.cache_read_tokens += stats.cache_read_tokens\n self.cache_creation_tokens += stats.cache_creation_tokens\n self.output_text_tokens += stats.output_text_tokens\n self.embedding_input_tokens += stats.embedding_input_tokens\n self.record_op_stats_lst.extend(stats.record_op_stats_lst)\n\n elif isinstance(stats, RecordOpStats):\n stats.source_unique_full_op_ids = self.source_unique_full_op_ids\n stats.plan_id = self.plan_id\n self.record_op_stats_lst.append(stats)\n self.total_op_time += stats.time_per_record\n self.total_op_cost += stats.cost_per_record\n self.input_text_tokens += stats.input_text_tokens\n self.input_audio_tokens += stats.input_audio_tokens\n self.input_image_tokens += stats.input_image_tokens\n self.cache_read_tokens += stats.cache_read_tokens\n self.cache_creation_tokens += stats.cache_creation_tokens\n self.output_text_tokens += stats.output_text_tokens\n self.embedding_input_tokens += stats.embedding_input_tokens\n\n else:\n raise TypeError(f\"Cannot add {type(stats)} to OperatorStats\")\n\n return self\n\n\nclass BasePlanStats(BaseModel):\n \"\"\"\n Model for storing statistics captured for an entire plan.\n\n This class is subclassed for tracking:\n - PlanStats: the statistics for execution of a PhysicalPlan\n - SentinelPlanStats: the statistics for execution of a SentinelPlan\n\n The key difference between the two subclasses is that the `operator_stats`\n field in the PlanStats maps from the physical operator ids to their corresponding\n OperatorStats objects.\n\n The `operator_stats` field in the SentinelPlanStats maps from a logical operator id\n to another dictionary which maps from the physical operator ids to their corresponding\n OperatorStats objects.\n \"\"\"\n\n # id for identifying the physical plan\n plan_id: str\n\n # string representation of the physical plan\n plan_str: str | None = None\n\n # dictionary whose values are OperatorStats objects;\n # PlanStats maps {full_op_id -> OperatorStats}\n # SentinelPlanStats maps {logical_op_id -> {full_op_id -> OperatorStats}}\n operator_stats: dict[str, OperatorStats | dict[str, OperatorStats]] = Field(default_factory=dict)\n\n # dictionary whose values are GenerationStats objects for validation;\n # only used by SentinelPlanStats\n validation_gen_stats: dict[str, GenerationStats] = Field(default_factory=dict)\n\n # total runtime for the plan measured from the start to the end of PhysicalPlan.execute()\n total_plan_time: float = 0.0\n\n # total cost for plan\n total_plan_cost: float = 0.0\n\n # input text tokens processed by this plan\n input_text_tokens: float = 0.0\n\n # input audio tokens processed by this plan\n input_audio_tokens: float = 0.0\n\n # input image tokens processed by this plan\n input_image_tokens: float = 0.0\n\n # cache read tokens processed by this plan\n cache_read_tokens: float = 0.0\n\n # tokens written to cache by this plan\n cache_creation_tokens: float = 0.0\n\n # output text tokens generated by this plan\n output_text_tokens: float = 0.0\n\n # embedding input tokens processed by this plan\n embedding_input_tokens: float = 0.0\n\n # start time for the plan execution; should be set by calling PlanStats.start()\n start_time: float | None = None\n\n def start(self) -> None:\n \"\"\"Start the timer for this plan execution.\"\"\"\n self.start_time = time.time()\n\n def finish(self) -> None:\n \"\"\"Finish the timer for this plan execution.\"\"\"\n if self.start_time is None:\n raise RuntimeError(\"PlanStats.start() must be called before PlanStats.finish()\")\n self.total_plan_time = time.time() - self.start_time\n self.total_plan_cost = self.sum_op_stats_field(\"total_op_cost\") + self.sum_validation_stats_field(\"cost_per_record\")\n self.input_text_tokens = self.sum_op_stats_field(\"input_text_tokens\") + self.sum_validation_stats_field(\"input_text_tokens\")\n self.input_audio_tokens = self.sum_op_stats_field(\"input_audio_tokens\") + self.sum_validation_stats_field(\"input_audio_tokens\")\n self.input_image_tokens = self.sum_op_stats_field(\"input_image_tokens\") + self.sum_validation_stats_field(\"input_image_tokens\")\n self.cache_read_tokens = self.sum_op_stats_field(\"cache_read_tokens\") + self.sum_validation_stats_field(\"cache_read_tokens\")\n self.cache_creation_tokens = self.sum_op_stats_field(\"cache_creation_tokens\") + self.sum_validation_stats_field(\"cache_creation_tokens\")\n self.output_text_tokens = self.sum_op_stats_field(\"output_text_tokens\") + self.sum_validation_stats_field(\"output_text_tokens\")\n self.embedding_input_tokens = self.sum_op_stats_field(\"embedding_input_tokens\") + self.sum_validation_stats_field(\"embedding_input_tokens\")\n\n @staticmethod\n @abstractmethod\n def from_plan(plan) -> BasePlanStats:\n \"\"\"\n Initialize this PlanStats object from a PhysicalPlan or SentinelPlan object.\n \"\"\"\n pass\n\n @abstractmethod\n def sum_op_stats_field(self, field_name: str) -> float | int:\n \"\"\"Sum a given field across all operator stats in this plan.\"\"\"\n pass\n\n def sum_validation_stats_field(self, field_name: str) -> float | int:\n \"\"\"Sum a given field across all validation generation stats in this plan.\"\"\"\n return sum([getattr(gen_stats, field_name) for _, gen_stats in self.validation_gen_stats.items()])\n\n @abstractmethod\n def add_record_op_stats(self, unique_full_op_id: str, record_op_stats: RecordOpStats | list[RecordOpStats]) -> None:\n \"\"\"\n Add the given RecordOpStats to this plan's operator stats for the given operator id.\n \"\"\"\n pass\n\n @abstractmethod\n def __iadd__(self, plan_stats: BasePlanStats) -> None:\n \"\"\"\n Add the given PlanStats to this plan's operator stats.\n \"\"\"\n pass\n\n @abstractmethod\n def __str__(self) -> str:\n \"\"\"\n Return a string representation of this plan's statistics.\n \"\"\"\n pass\n\n def get_total_cost_so_far(self) -> float:\n \"\"\"\n Get the total cost incurred so far in this plan execution.\n \"\"\"\n return self.sum_op_stats_field(\"total_op_cost\") + self.sum_validation_stats_field(\"cost_per_record\")\n\n\nclass PlanStats(BasePlanStats):\n \"\"\"\n Subclass of BasePlanStats which captures statistics from the execution of a single PhysicalPlan.\n \"\"\"\n @staticmethod\n def from_plan(plan) -> PlanStats:\n \"\"\"\n Initialize this PlanStats object from a PhysicalPlan object.\n \"\"\"\n # TODO?: have PhysicalPlan return PlanStats object\n operator_stats = {}\n for topo_idx, op in enumerate(plan):\n unique_full_op_id = f\"{topo_idx}-{op.get_full_op_id()}\"\n operator_stats[unique_full_op_id] = OperatorStats(\n full_op_id=op.get_full_op_id(),\n op_name=op.op_name(),\n source_unique_full_op_ids=plan.get_source_unique_full_op_ids(topo_idx, op),\n plan_id=plan.plan_id,\n op_details={k: str(v) for k, v in op.get_id_params().items()},\n )\n\n return PlanStats(plan_id=plan.plan_id, plan_str=str(plan), operator_stats=operator_stats)\n \n def sum_op_stats_field(self, field_name: str) -> float | int:\n \"\"\"Sum a given field across all operator stats in this plan.\"\"\"\n return sum([getattr(op_stats, field_name) for _, op_stats in self.operator_stats.items()])\n\n def add_record_op_stats(self, unique_full_op_id: str, record_op_stats: RecordOpStats | list[RecordOpStats]) -> None:\n \"\"\"\n Add the given RecordOpStats to this plan's operator stats for the given operator id.\n \"\"\"\n # normalize input type to be list[RecordOpStats]\n record_op_stats_lst = record_op_stats if isinstance(record_op_stats, list) else [record_op_stats]\n\n # update operator stats\n for record_op_stats in record_op_stats_lst:\n if unique_full_op_id in self.operator_stats:\n self.operator_stats[unique_full_op_id] += record_op_stats\n else:\n raise ValueError(f\"RecordOpStats with unique_full_op_id {unique_full_op_id} not found in PlanStats\")\n\n def __iadd__(self, plan_stats: PlanStats) -> None:\n \"\"\"\n NOTE: we assume the execution layer guarantees:\n 1. these plan_stats belong to the same plan\n 2. these plan_stats come from sequential (non-overlapping) executions of the same plan\n\n The latter criteria implies it is okay for this method to sum the plan (and operator) runtimes.\n \"\"\"\n self.total_plan_time += plan_stats.total_plan_time\n self.total_plan_cost += plan_stats.total_plan_cost\n self.input_text_tokens += plan_stats.input_text_tokens\n self.input_audio_tokens += plan_stats.input_audio_tokens\n self.input_image_tokens += plan_stats.input_image_tokens\n self.cache_read_tokens += plan_stats.cache_read_tokens\n self.cache_creation_tokens += plan_stats.cache_creation_tokens\n self.output_text_tokens += plan_stats.output_text_tokens\n self.embedding_input_tokens += plan_stats.embedding_input_tokens\n for unique_full_op_id, op_stats in plan_stats.operator_stats.items():\n if unique_full_op_id in self.operator_stats:\n self.operator_stats[unique_full_op_id] += op_stats\n else:\n self.operator_stats[unique_full_op_id] = op_stats\n\n def __str__(self) -> str:\n stats = f\"total_plan_time={self.total_plan_time} \\n\"\n stats += f\"total_plan_cost={self.total_plan_cost} \\n\"\n stats += f\"input_text_tokens={self.input_text_tokens} \\n\"\n stats += f\"input_audio_tokens={self.input_audio_tokens} \\n\"\n stats += f\"input_image_tokens={self.input_image_tokens} \\n\"\n stats += f\"cache_read_tokens={self.cache_read_tokens} \\n\"\n stats += f\"cache_creation_tokens={self.cache_creation_tokens} \\n\"\n stats += f\"output_text_tokens={self.output_text_tokens} \\n\"\n stats += f\"embedding_input_tokens={self.embedding_input_tokens} \\n\"\n for idx, op_stats in enumerate(self.operator_stats.values()):\n stats += f\"{idx}. {op_stats.op_name} time={op_stats.total_op_time} cost={op_stats.total_op_cost} \\n\"\n return stats\n\n\nclass SentinelPlanStats(BasePlanStats):\n \"\"\"\n Subclass of BasePlanStats which captures statistics from the execution of a single SentinelPlan.\n \"\"\"\n @staticmethod\n def from_plan(plan) -> SentinelPlanStats:\n \"\"\"\n Initialize this PlanStats object from a Sentinel object.\n \"\"\"\n operator_stats = {}\n for topo_idx, (logical_op_id, op_set) in enumerate(plan):\n unique_logical_op_id = f\"{topo_idx}-{logical_op_id}\"\n operator_stats[unique_logical_op_id] = {}\n for physical_op in op_set:\n full_op_id = physical_op.get_full_op_id()\n operator_stats[unique_logical_op_id][full_op_id] = OperatorStats(\n full_op_id=full_op_id,\n op_name=physical_op.op_name(),\n source_unique_logical_op_ids=plan.get_source_unique_logical_op_ids(unique_logical_op_id),\n plan_id=plan.plan_id,\n op_details={k: str(v) for k, v in physical_op.get_id_params().items()},\n )\n\n return SentinelPlanStats(plan_id=plan.plan_id, plan_str=str(plan), operator_stats=operator_stats)\n\n def sum_op_stats_field(self, field_name: str) -> float | int:\n \"\"\"Sum a given field across all operator stats in this plan.\"\"\"\n return sum(sum([getattr(op_stats, field_name) for _, op_stats in phys_op_stats.items()]) for _, phys_op_stats in self.operator_stats.items())\n\n def add_record_op_stats(self, unique_logical_op_id: str, record_op_stats: RecordOpStats | list[RecordOpStats]) -> None:\n \"\"\"\n Add the given RecordOpStats to this plan's operator stats for the given operator set id.\n \"\"\"\n # normalize input type to be list[RecordOpStats]\n record_op_stats_lst = record_op_stats if isinstance(record_op_stats, list) else [record_op_stats]\n\n # update operator stats\n for record_op_stats in record_op_stats_lst:\n full_op_id = record_op_stats.full_op_id\n if unique_logical_op_id in self.operator_stats:\n if full_op_id in self.operator_stats[unique_logical_op_id]:\n self.operator_stats[unique_logical_op_id][full_op_id] += record_op_stats\n else:\n raise ValueError(f\"RecordOpStats with full_op_id {full_op_id} not found in SentinelPlanStats\")\n else:\n raise ValueError(f\"RecordOpStats with unique_logical_op_id {unique_logical_op_id} not found in SentinelPlanStats\")\n\n def add_validation_gen_stats(self, unique_logical_op_id: str, gen_stats: GenerationStats) -> None:\n \"\"\"\n Add the given GenerationStats to this plan's validation generation stats for the given logical operator id.\n \"\"\"\n if unique_logical_op_id in self.validation_gen_stats:\n self.validation_gen_stats[unique_logical_op_id] += gen_stats\n else:\n self.validation_gen_stats[unique_logical_op_id] = gen_stats\n\n def __iadd__(self, plan_stats: SentinelPlanStats) -> None:\n \"\"\"\n NOTE: we assume the execution layer guarantees:\n 1. these plan_stats belong to the same plan\n 2. these plan_stats come from sequential (non-overlapping) executions of the same plan\n\n The latter criteria implies it is okay for this method to sum the plan (and operator) runtimes.\n \"\"\"\n self.total_plan_time += plan_stats.total_plan_time\n self.total_plan_cost += plan_stats.total_plan_cost\n self.input_text_tokens += plan_stats.input_text_tokens\n self.input_audio_tokens += plan_stats.input_audio_tokens\n self.input_image_tokens += plan_stats.input_image_tokens\n self.cache_read_tokens += plan_stats.cache_read_tokens\n self.cache_creation_tokens += plan_stats.cache_creation_tokens\n self.output_text_tokens += plan_stats.output_text_tokens\n self.embedding_input_tokens += plan_stats.embedding_input_tokens\n for unique_logical_op_id, physical_op_stats in plan_stats.operator_stats.items():\n for full_op_id, op_stats in physical_op_stats.items():\n if unique_logical_op_id in self.operator_stats:\n if full_op_id in self.operator_stats[unique_logical_op_id]:\n self.operator_stats[unique_logical_op_id][full_op_id] += op_stats\n else:\n self.operator_stats[unique_logical_op_id][full_op_id] = op_stats\n else:\n self.operator_stats[unique_logical_op_id] = physical_op_stats\n\n for unique_logical_op_id, gen_stats in plan_stats.validation_gen_stats.items():\n if unique_logical_op_id in self.validation_gen_stats:\n self.validation_gen_stats[unique_logical_op_id] += gen_stats\n else:\n self.validation_gen_stats[unique_logical_op_id] = gen_stats\n\n def __str__(self) -> str:\n stats = f\"total_plan_time={self.total_plan_time} \\n\"\n stats += f\"total_plan_cost={self.total_plan_cost} \\n\"\n stats += f\"input_text_tokens={self.input_text_tokens} \\n\"\n stats += f\"input_audio_tokens={self.input_audio_tokens} \\n\"\n stats += f\"input_image_tokens={self.input_image_tokens} \\n\"\n stats += f\"cache_read_tokens={self.cache_read_tokens} \\n\"\n stats += f\"cache_creation_tokens={self.cache_creation_tokens} \\n\"\n stats += f\"output_text_tokens={self.output_text_tokens} \\n\"\n stats += f\"embedding_input_tokens={self.embedding_input_tokens} \\n\"\n for outer_idx, physical_op_stats in enumerate(self.operator_stats.values()):\n total_time = sum([op_stats.total_op_time for op_stats in physical_op_stats.values()])\n total_cost = sum([op_stats.total_op_cost for op_stats in physical_op_stats.values()])\n stats += f\"{outer_idx}. total_time={total_time} total_cost={total_cost} \\n\"\n for inner_idx, op_stats in enumerate(physical_op_stats.values()):\n stats += f\" {outer_idx}.{inner_idx}. {op_stats.op_name} time={op_stats.total_op_time} cost={op_stats.total_op_cost} \\n\"\n return stats\n\n\nclass ExecutionStats(BaseModel):\n \"\"\"\n Model for storing statistics captured for the entire execution of a workload.\n \"\"\"\n\n # string for identifying this workload execution\n execution_id: str | None = None\n\n # dictionary of SentinelPlanStats objects (one for each sentinel plan run during execution)\n sentinel_plan_stats: dict[str, SentinelPlanStats] = Field(default_factory=dict)\n\n # dictionary of PlanStats objects (one for each plan run during execution)\n plan_stats: dict[str, PlanStats] = Field(default_factory=dict)\n\n # total time spent optimizing\n optimization_time: float = 0.0\n\n # total cost of optimizing\n optimization_cost: float = 0.0\n\n # total time spent executing the optimized plan\n plan_execution_time: float = 0.0\n\n # total cost of executing the optimized plan\n plan_execution_cost: float = 0.0\n\n # total runtime for the entire execution\n total_execution_time: float = 0.0\n\n # total cost for the entire execution\n total_execution_cost: float = 0.0\n\n # input text tokens processed\n input_text_tokens: float = 0.0\n\n # input audio tokens processed\n input_audio_tokens: float = 0.0\n\n # input image tokens processed\n input_image_tokens: float = 0.0\n\n # cache read tokens processed\n cache_read_tokens: float = 0.0\n\n # tokens written to cache\n cache_creation_tokens: float = 0.0\n\n # output text tokens generated\n output_text_tokens: float = 0.0\n\n # embedding input tokens processed\n embedding_input_tokens: float = 0.0\n\n # dictionary of sentinel plan strings; useful for printing executed sentinel plans in demos\n sentinel_plan_strs: dict[str, str] = Field(default_factory=dict)\n\n # dictionary of plan strings; useful for printing executed plans in demos\n plan_strs: dict[str, str] = Field(default_factory=dict)\n\n # start time for the execution; should be set by calling ExecutionStats.start()\n start_time: float | None = None\n\n # end time for the optimization;\n optimization_end_time: float | None = None\n\n def start(self) -> None:\n \"\"\"Start the timer for this execution.\"\"\"\n self.start_time = time.time()\n\n def finish_optimization(self) -> None:\n \"\"\"Finish the timer for the optimization phase of this execution.\"\"\"\n if self.start_time is None:\n raise RuntimeError(\"ExecutionStats.start() must be called before ExecutionStats.finish_optimization()\")\n\n # compute optimization time and cost\n self.optimization_end_time = time.time()\n self.optimization_time = self.optimization_end_time - self.start_time\n self.optimization_cost = self.sum_sentinel_plan_costs()\n\n # compute sentinel_plan_strs\n self.sentinel_plan_strs = {plan_id: plan_stats.plan_str for plan_id, plan_stats in self.sentinel_plan_stats.items()}\n\n def finish(self) -> None:\n \"\"\"Finish the timer for this execution.\"\"\"\n if self.start_time is None:\n raise RuntimeError(\"ExecutionStats.start() must be called before ExecutionStats.finish()\")\n\n # compute time for plan and total execution\n end_time = time.time()\n self.plan_execution_time = (\n end_time - self.optimization_end_time\n if self.optimization_end_time is not None\n else end_time - self.start_time\n )\n self.total_execution_time = end_time - self.start_time\n\n # compute the cost for plan and total execution\n self.plan_execution_cost = self.sum_plan_costs()\n self.total_execution_cost = self.optimization_cost + self.plan_execution_cost\n\n # compute the tokens for total execution\n self.input_text_tokens = self.sum_plan_stats_field(\"input_text_tokens\")\n self.input_audio_tokens = self.sum_plan_stats_field(\"input_audio_tokens\")\n self.input_image_tokens = self.sum_plan_stats_field(\"input_image_tokens\")\n self.cache_read_tokens = self.sum_plan_stats_field(\"cache_read_tokens\")\n self.cache_creation_tokens = self.sum_plan_stats_field(\"cache_creation_tokens\")\n self.output_text_tokens = self.sum_plan_stats_field(\"output_text_tokens\")\n self.embedding_input_tokens = self.sum_plan_stats_field(\"embedding_input_tokens\")\n\n # compute plan_strs\n self.plan_strs = {plan_id: plan_stats.plan_str for plan_id, plan_stats in self.plan_stats.items()}\n\n def sum_plan_stats_field(self, field_name: str) -> float | int:\n \"\"\"\n Sum a given field across all PlanStats in this execution.\n \"\"\"\n sentinel_plan_field_sum = sum([plan_stats.sum_op_stats_field(field_name) + plan_stats.sum_validation_stats_field(field_name) for _, plan_stats in self.sentinel_plan_stats.items()])\n plan_field_sum = sum([plan_stats.sum_op_stats_field(field_name) for _, plan_stats in self.plan_stats.items()])\n return plan_field_sum + sentinel_plan_field_sum\n\n def sum_sentinel_plan_costs(self) -> float:\n \"\"\"\n Sum the costs of all SentinelPlans in this execution.\n \"\"\"\n return sum([\n plan_stats.sum_op_stats_field(\"total_op_cost\") + plan_stats.sum_validation_stats_field(\"cost_per_record\")\n for _, plan_stats in self.sentinel_plan_stats.items()\n ])\n\n def sum_plan_costs(self) -> float:\n \"\"\"\n Sum the costs of all PhysicalPlans in this execution.\n \"\"\"\n return sum([plan_stats.sum_op_stats_field(\"total_op_cost\") for _, plan_stats in self.plan_stats.items()])\n\n def add_plan_stats(self, plan_stats: PlanStats | SentinelPlanStats | list[PlanStats] | list[SentinelPlanStats]) -> None:\n \"\"\"\n Add the given PlanStats (or SentinelPlanStats) to this execution's plan stats.\n\n NOTE: we make the assumption that the same plan cannot be run more than once in parallel,\n i.e. each plan stats object for an individual plan comes from two different (sequential)\n periods in time. Thus, PlanStats objects can be summed.\n \"\"\"\n # normalize input type to be list[PlanStats] or list[SentinelPlanStats]\n if isinstance(plan_stats, (PlanStats, SentinelPlanStats)):\n plan_stats = [plan_stats]\n\n for plan_stats_obj in plan_stats:\n if isinstance(plan_stats_obj, PlanStats) and plan_stats_obj.plan_id not in self.plan_stats:\n self.plan_stats[plan_stats_obj.plan_id] = plan_stats_obj\n elif isinstance(plan_stats_obj, PlanStats):\n self.plan_stats[plan_stats_obj.plan_id] += plan_stats_obj\n elif isinstance(plan_stats_obj, SentinelPlanStats) and plan_stats_obj.plan_id not in self.sentinel_plan_stats:\n self.sentinel_plan_stats[plan_stats_obj.plan_id] = plan_stats_obj\n elif isinstance(plan_stats_obj, SentinelPlanStats):\n self.sentinel_plan_stats[plan_stats_obj.plan_id] += plan_stats_obj\n else:\n raise TypeError(f\"Cannot add {type(plan_stats)} to ExecutionStats\")\n\n def to_json(self, filepath: str | None = None) -> dict | None:\n if filepath is None:\n return self.model_dump(mode=\"json\")\n\n with open(filepath, \"w\") as f:\n json.dump(self.model_dump(mode=\"json\"), f)\n\n\nclass OperatorCostEstimates(BaseModel):\n \"\"\"\n Model for storing estimates of key metrics of interest for each operator.\n \"\"\"\n\n # (estimated) number of records output by this operator\n cardinality: float\n\n # (estimated) avg. time spent in this operator per-record\n time_per_record: float\n\n # (estimated) dollars spent per-record by this operator\n cost_per_record: float\n\n # (estimated) quality of the output from this operator\n quality: float\n\n # lower bound on cardinality\n cardinality_lower_bound: float | None = None\n\n # upper bound on cardinality\n cardinality_upper_bound: float | None = None\n\n # lower bound on time_per_record\n time_per_record_lower_bound: float | None = None\n\n # upper bound on time_per_record\n time_per_record_upper_bound: float | None = None\n\n # lower bound on cost_per_record\n cost_per_record_lower_bound: float | None = None\n\n # upper bound on cost_per_record\n cost_per_record_upper_bound: float | None = None\n\n # lower bound on quality\n quality_lower_bound: float | None = None\n\n # upper bound on quality\n quality_upper_bound: float | None = None\n\n def __rmul__(self, multiplier: float) -> OperatorCostEstimates:\n \"\"\"\n Multiply all fields by a scalar.\n \"\"\"\n dct = {field_name: getattr(self, field_name) * multiplier for field_name in self.model_fields}\n return OperatorCostEstimates(**dct)\n\n def model_post_init(self, __context: Any) -> None:\n if self.cardinality_lower_bound is None and self.cardinality_upper_bound is None:\n self.cardinality_lower_bound = self.cardinality\n self.cardinality_upper_bound = self.cardinality\n\n if self.time_per_record_lower_bound is None and self.time_per_record_upper_bound is None:\n self.time_per_record_lower_bound = self.time_per_record\n self.time_per_record_upper_bound = self.time_per_record\n\n if self.cost_per_record_lower_bound is None and self.cost_per_record_upper_bound is None:\n self.cost_per_record_lower_bound = self.cost_per_record\n self.cost_per_record_upper_bound = self.cost_per_record\n\n if self.quality_lower_bound is None and self.quality_upper_bound is None:\n self.quality_lower_bound = self.quality\n self.quality_upper_bound = self.quality\n\n\nclass PlanCost(BaseModel):\n \"\"\"\n Model for storing the (cost, time, quality) estimates of (sub)-plans and their upper and lower bounds.\n \"\"\"\n\n # the expression cost\n cost: float\n\n # the expression runtime\n time: float\n\n # the expression quality\n quality: float\n\n # operator-specific cost estimates\n op_estimates: OperatorCostEstimates | None = None\n\n # lower bound on the expression cost\n cost_lower_bound: float | None = None\n\n # upper bound on the expression cost\n cost_upper_bound: float | None = None\n\n # lower bound on the expression time\n time_lower_bound: float | None = None\n\n # upper bound on the expression time\n time_upper_bound: float | None = None\n\n # lower bound on the expression quality\n quality_lower_bound: float | None = None\n\n # upper bound on the expression quality\n quality_upper_bound: float | None = None\n\n def __hash__(self):\n return hash(f\"{self.cost}-{self.time}-{self.quality}\")\n\n def __eq__(self, other: Any) -> bool:\n if not isinstance(other, PlanCost):\n return False\n return (\n self.cost == other.cost\n and self.time == other.time\n and self.quality == other.quality\n )\n\n def model_post_init(self, __context: Any) -> None:\n if self.time_lower_bound is None and self.time_upper_bound is None:\n self.time_lower_bound = self.time\n self.time_upper_bound = self.time\n\n if self.cost_lower_bound is None and self.cost_upper_bound is None:\n self.cost_lower_bound = self.cost\n self.cost_upper_bound = self.cost\n\n if self.quality_lower_bound is None and self.quality_upper_bound is None:\n self.quality_lower_bound = self.quality\n self.quality_upper_bound = self.quality\n\n def join_add(self, left_plan_cost: PlanCost, right_plan_cost: PlanCost, execution_strategy: str = \"parallel\") -> PlanCost:\n \"\"\"\n Add the PlanCost objects for two joined plans (left_plan_cost and right_plan_cost)\n to the PlanCost object for the join operator. The execution strategy determines how\n the input times are combined. If the execution strategy is \"parallel\", the input time\n is the maximum of the two times. If the execution strategy is \"sequential\" (which is\n currently anything else), the input time is the sum of the two times.\n\n For quality, we compute the produce of the operator quality with the average of the\n two input qualities.\n\n NOTE: we currently assume the updating of the op_estimates are handled by the caller\n as there is not a universally correct meaning of addition of op_estimates.\n \"\"\"\n dct = {}\n for model_field in [\"cost\", \"cost_lower_bound\", \"cost_upper_bound\"]:\n op_field_value = getattr(self, model_field)\n left_plan_field_value = getattr(left_plan_cost, model_field)\n right_plan_field_value = getattr(right_plan_cost, model_field)\n if op_field_value is not None and left_plan_field_value is not None and right_plan_field_value is not None:\n dct[model_field] = op_field_value + left_plan_field_value + right_plan_field_value\n\n for model_field in [\"time\", \"time_lower_bound\", \"time_upper_bound\"]:\n op_field_value = getattr(self, model_field)\n left_plan_field_value = getattr(left_plan_cost, model_field)\n right_plan_field_value = getattr(right_plan_cost, model_field)\n if op_field_value is not None and left_plan_field_value is not None and right_plan_field_value is not None:\n if execution_strategy == \"parallel\":\n dct[model_field] = op_field_value + max(left_plan_field_value, right_plan_field_value)\n else:\n dct[model_field] = op_field_value + left_plan_field_value + right_plan_field_value\n\n for model_field in [\"quality\", \"quality_lower_bound\", \"quality_upper_bound\"]:\n op_field_value = getattr(self, model_field)\n left_plan_field_value = getattr(left_plan_cost, model_field)\n right_plan_field_value = getattr(right_plan_cost, model_field)\n if op_field_value is not None and left_plan_field_value is not None and right_plan_field_value is not None:\n dct[model_field] = op_field_value * ((left_plan_field_value + right_plan_field_value) / 2.0)\n\n return PlanCost(**dct)\n\n def __iadd__(self, other: PlanCost) -> PlanCost:\n \"\"\"\n NOTE: we currently assume the updating of the op_estimates are handled by the caller\n as there is not a universally correct meaning of addition of op_estimates.\n \"\"\"\n self.cost += other.cost\n self.time += other.time\n self.quality *= other.quality\n for model_field in [\"cost_lower_bound\", \"cost_upper_bound\", \"time_lower_bound\", \"time_upper_bound\"]:\n if getattr(self, model_field) is not None and getattr(other, model_field) is not None:\n summation = getattr(self, model_field) + getattr(other, model_field)\n setattr(self, model_field, summation)\n\n for model_field in [\"quality_lower_bound\", \"quality_upper_bound\"]:\n if getattr(self, model_field) is not None and getattr(other, model_field) is not None:\n product = getattr(self, model_field) * getattr(other, model_field)\n setattr(self, model_field, product)\n\n return self\n\n def __add__(self, other: PlanCost) -> PlanCost:\n \"\"\"\n NOTE: we currently assume the updating of the op_estimates are handled by the caller\n as there is not a universally correct meaning of addition of op_estimates.\n \"\"\"\n dct = {\n field: getattr(self, field) + getattr(other, field)\n for field in [\n \"cost\",\n \"cost_lower_bound\",\n \"cost_upper_bound\",\n \"time\",\n \"time_lower_bound\",\n \"time_upper_bound\",\n ]\n }\n for model_field in [\"quality\", \"quality_lower_bound\", \"quality_upper_bound\"]:\n dct[model_field] = getattr(self, model_field) * getattr(other, model_field)\n\n return PlanCost(**dct)\n" }, { "path": "src/palimpzest/policy.py", "content": "from __future__ import annotations\n\nimport json\n\nfrom palimpzest.core.models import PlanCost\n\n\ndef construct_policy_from_kwargs(**kwargs) -> Policy | None:\n \"\"\"\n Construct and return a policy object which is defined by the keyword arguments.\n\n This function accepts the following keyword arguments:\n - max_quality\n - min_cost\n - min_time\n - cost_budget\n - time_budget\n - quality_threshold\n\n If none of these keyword arguments are provided, the function will return None.\n \"\"\"\n # compute the number of objectives and constraints in the kwargs\n num_objectives = sum([bool(kwargs.get(key, False)) for key in [\"max_quality\", \"min_cost\", \"min_time\"]])\n num_constraints = sum([bool(kwargs.get(key, False)) for key in [\"cost_budget\", \"time_budget\", \"quality_threshold\"]])\n\n # if there are no policy kwargs provided, return None\n if num_objectives == 0 and num_constraints == 0:\n return None\n\n # Otherwise, assert that kwargs are valid\n assert num_objectives == 1, \"Must optimize for one of max_quality, min_cost, or min_time.\"\n assert num_constraints <= 1, \"Currently, PZ supports at most one constraint.\"\n\n # print warning if user tries to set a constraint and optimization goal on the same metric\n if \"max_quality\" in kwargs and \"quality_threshold\" in kwargs:\n print(\"Warning: Setting a constraint on quality and optimizing for quality is redundant.\")\n\n if \"min_cost\" in kwargs and \"cost_budget\" in kwargs:\n print(\"Warning: Setting a constraint on cost and optimizing for cost is redundant.\")\n\n if \"min_time\" in kwargs and \"time_budget\" in kwargs:\n print(\"Warning: Setting a constraint on time and optimizing for time is redundant.\")\n\n # construct the policy object\n policy = None\n if \"max_quality\" in kwargs and \"cost_budget\" in kwargs:\n policy = MaxQualityAtFixedCost(kwargs[\"cost_budget\"])\n elif \"max_quality\" in kwargs and \"time_budget\" in kwargs:\n policy = MaxQualityAtFixedTime(kwargs[\"time_budget\"])\n elif \"max_quality\" in kwargs:\n policy = MaxQuality()\n elif \"min_cost\" in kwargs and \"quality_threshold\" in kwargs:\n policy = MinCostAtFixedQuality(kwargs[\"quality_threshold\"])\n elif \"min_cost\" in kwargs:\n policy = MinCost()\n elif \"min_time\" in kwargs and \"quality_threshold\" in kwargs:\n policy = MinTimeAtFixedQuality(kwargs[\"quality_threshold\"])\n elif \"min_time\" in kwargs:\n policy = MinTime()\n\n return policy\n\n\nclass Policy:\n \"\"\"\n Base class for a policy. Each policy has two methods: constraint() and chooose().\n The first method determines whether the given cost, runtime, and quality for a plan\n (or sub-plan) satisfy the policy's constraint(s). The second method takes in the \n (cost, runtime, quality) tuples for two plans (or subplans) and returns True if the\n first plan is better than the second one and False otherwise.\n \"\"\"\n\n def __init__(self):\n pass\n\n def get_primary_metric(self) -> str:\n \"\"\"\n Returns one of [\"cost\", \"time\", \"quality\"]; whichever corresponds to the\n maximization / minimization goal of the policy.\n\n Eventually we may make policies more general by allowing users to optimize\n some function: f(cost, time, quality). In that case, we may deprecate this\n method and update its callers.\n \"\"\"\n raise NotImplementedError(\"Calling this method from an abstract base class.\")\n\n def get_dict(self) -> dict:\n \"\"\"\n Returns a dict representation of the policy which specifies how much weight\n (in [0,1]) should be given to each metric.\n \"\"\"\n raise NotImplementedError(\"Calling this method from an abstract base class.\")\n\n def constraint(self, plan: PlanCost) -> bool:\n \"\"\"\n Return True if the given (cost, runtime, quality) for a plan (or subplan)\n satisfy the policy's constraint(s). Otherwise, return False.\n \"\"\"\n raise NotImplementedError(\"Calling this method from an abstract base class.\")\n\n def choose(self, plan: PlanCost, other_plan: PlanCost) -> float:\n \"\"\"\n Return True if plan is better than other_plan and return False otherwise.\n \"\"\"\n raise NotImplementedError(\"Calling this method from an abstract base class.\")\n\n def to_json_str(self) -> str:\n \"\"\"Convert policy configuration to a JSON-serializable dictionary.\"\"\"\n return json.dumps({\n \"type\": self.__class__.__name__,\n \"config\": self.get_dict()\n }, indent=2)\n\n\nclass MaxQuality(Policy):\n \"\"\"\n This policy has no constraints and computes the best plan as the one with\n the higher quality.\n \"\"\"\n\n def __str__(self):\n return \"Maximum Quality\"\n\n def get_primary_metric(self) -> str:\n return \"quality\"\n\n def get_dict(self) -> dict:\n return {\"cost\": 0.0, \"time\": 0.0, \"quality\": 1.0}\n\n def constraint(self, plan: PlanCost) -> bool:\n \"\"\"There is no constraint.\"\"\"\n return True\n\n def choose(self, plan: PlanCost, other_plan: PlanCost) -> float:\n \"\"\"\n Return True if plan has higher quality than other_plan and return False otherwise.\n Use cost and then runtime as tiebreakers.\n \"\"\"\n if plan.quality == other_plan.quality:\n if plan.cost == other_plan.cost:\n return plan.time < other_plan.time\n return plan.cost < other_plan.cost\n\n return plan.quality > other_plan.quality\n\n\nclass MinCost(Policy):\n \"\"\"\n This policy has no constraints and computes the best plan as the one with\n the lower cost.\n \"\"\"\n\n def __str__(self):\n return \"Minimum Cost\"\n\n def get_primary_metric(self) -> str:\n return \"cost\"\n\n def get_dict(self) -> dict:\n return {\"cost\": 1.0, \"time\": 0.0, \"quality\": 0.0}\n\n def constraint(self, plan: PlanCost) -> bool:\n \"\"\"There is no constraint.\"\"\"\n return True\n\n def choose(self, plan: PlanCost, other_plan: PlanCost) -> float:\n \"\"\"\n Return True if plan has lower cost than other_plan and return False otherwise.\n Use quality and then runtime as tiebreakers.\n \"\"\"\n if plan.cost == other_plan.cost:\n if plan.quality == other_plan.quality:\n return plan.time < other_plan.time\n return plan.quality > other_plan.quality\n\n return plan.cost < other_plan.cost\n\n\nclass MinTime(Policy):\n \"\"\"\n This policy has no constraints and computes the best plan as the one with\n the lower runtime.\n \"\"\"\n\n def __str__(self):\n return \"Minimum Time\"\n\n def get_primary_metric(self) -> str:\n return \"time\"\n\n def get_dict(self) -> dict:\n return {\"cost\": 0.0, \"time\": 1.0, \"quality\": 0.0}\n\n def constraint(self, plan: PlanCost) -> bool:\n \"\"\"There is no constraint.\"\"\"\n return True\n\n def choose(self, plan: PlanCost, other_plan: PlanCost) -> float:\n \"\"\"\n Return True if plan has lower runtime than other_plan and return False otherwise.\n Use quality and then cost as tiebreakers.\n \"\"\"\n if plan.time == other_plan.time:\n if plan.quality == other_plan.quality:\n return plan.cost < other_plan.cost\n return plan.quality > other_plan.quality\n\n return plan.time < other_plan.time\n\n\nclass MaxQualityAtFixedCost(Policy):\n \"\"\"\n This policy applies a constraint (upper bound) on the cost of the plan\n and tries to maximize quality subject to that constraint.\n \"\"\"\n\n def __init__(self, max_cost: float):\n self.max_cost = max_cost\n\n def __str__(self):\n return \"MaxQuality@FixedCost\"\n\n def get_primary_metric(self) -> str:\n return \"quality\"\n\n def get_dict(self) -> dict:\n return {\"cost\": 0.5, \"time\": 0.0, \"quality\": 0.5}\n\n def constraint(self, plan: PlanCost) -> bool:\n return plan.cost < self.max_cost\n\n def choose(self, plan: PlanCost, other_plan: PlanCost) -> float:\n \"\"\"\n Return True if plan has higher quality than other_plan and return False otherwise.\n Use cost and then runtime as a tie-breaker.\n \"\"\"\n if plan.quality == other_plan.quality:\n if plan.cost == other_plan.cost:\n return plan.time < other_plan.time\n return plan.cost < other_plan.cost\n\n return plan.quality > other_plan.quality\n\n\nclass MaxQualityAtFixedTime(Policy):\n \"\"\"\n This policy applies a constraint (upper bound) on the runtime of the plan\n and tries to maximize quality subject to that constraint.\n \"\"\"\n\n def __init__(self, max_time: float):\n self.max_time = max_time\n\n def __str__(self):\n return \"MaxQuality@FixedTime\"\n\n def get_primary_metric(self) -> str:\n return \"quality\"\n\n def get_dict(self) -> dict:\n return {\"cost\": 0.0, \"time\": 0.5, \"quality\": 0.5}\n\n def constraint(self, plan: PlanCost) -> bool:\n return plan.time < self.max_time\n\n def choose(self, plan: PlanCost, other_plan: PlanCost) -> float:\n \"\"\"\n Return True if plan has higher quality than other_plan and return False otherwise.\n Use runtime and then cost as a tie-breaker.\n \"\"\"\n if plan.quality == other_plan.quality:\n if plan.time == other_plan.time:\n return plan.cost < other_plan.cost\n return plan.time < other_plan.time\n\n return plan.quality > other_plan.quality\n\n\nclass MinCostAtFixedQuality(Policy):\n \"\"\"\n This policy applies a constraint (lower bound) on the quality of the plan\n and tries to minimize cost subject to that constraint.\n \"\"\"\n\n def __init__(self, min_quality: float):\n self.min_quality = min_quality\n\n def __str__(self):\n return \"MinCost@FixedQuality\"\n\n def get_primary_metric(self) -> str:\n return \"cost\"\n\n def get_dict(self) -> dict:\n return {\"cost\": 0.5, \"time\": 0.0, \"quality\": 0.5}\n\n def constraint(self, plan: PlanCost) -> bool:\n return plan.quality > self.min_quality\n\n def choose(self, plan: PlanCost, other_plan: PlanCost) -> float:\n \"\"\"\n Return True if plan has lower cost than other_plan and return False otherwise.\n Use quality and then runtime as a tie-breaker.\n \"\"\"\n if plan.cost == other_plan.cost:\n if plan.quality == other_plan.quality:\n return plan.time < other_plan.time\n return plan.quality > other_plan.quality\n\n return plan.cost < other_plan.cost\n\n\nclass MinTimeAtFixedQuality(Policy):\n \"\"\"\n This policy applies a constraint (lower bound) on the quality of the plan\n and tries to minimize runtime subject to that constraint.\n \"\"\"\n\n def __init__(self, min_quality: float):\n self.min_quality = min_quality\n\n def __str__(self):\n return \"MinTime@FixedQuality\"\n\n def get_primary_metric(self) -> str:\n return \"time\"\n\n def get_dict(self) -> dict:\n return {\"cost\": 0.0, \"time\": 0.5, \"quality\": 0.5}\n\n def constraint(self, plan: PlanCost) -> bool:\n return plan.quality > self.min_quality\n\n def choose(self, plan: PlanCost, other_plan: PlanCost) -> float:\n \"\"\"\n Return True if plan has lower runtime than other_plan and return False otherwise.\n Use quality and then cost as a tie-breaker.\n \"\"\"\n if plan.time == other_plan.time:\n if plan.quality == other_plan.quality:\n return plan.cost < other_plan.cost\n return plan.quality > other_plan.quality\n\n return plan.time < other_plan.time\n" }, { "path": "src/palimpzest/prompts/__init__.py", "content": "from palimpzest.prompts.agent_prompts import (\n CODE_AGENT_SYSTEM_PROMPT,\n DATA_DISCOVERY_AGENT_INITIAL_PLAN_PROMPT,\n DATA_DISCOVERY_AGENT_REPORT_PROMPT,\n DATA_DISCOVERY_AGENT_TASK_PROMPT,\n DATA_DISCOVERY_AGENT_UPDATE_PLAN_POST_MESSAGES_PROMPT,\n DATA_DISCOVERY_AGENT_UPDATE_PLAN_PRE_MESSAGES_PROMPT,\n FINAL_ANSWER_POST_MESSAGES_PROMPT,\n FINAL_ANSWER_PRE_MESSAGES_PROMPT,\n)\nfrom palimpzest.prompts.context_search import CONTEXT_SEARCH_PROMPT\nfrom palimpzest.prompts.prompt_factory import PromptFactory\nfrom palimpzest.prompts.prompt_manager import PromptManager\nfrom palimpzest.prompts.utils import (\n ONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION,\n ONE_TO_ONE_OUTPUT_FORMAT_INSTRUCTION,\n)\nfrom palimpzest.prompts.validator import (\n FLAT_MAP_IMAGE_VALIDATOR_PROMPT,\n FLAT_MAP_VALIDATOR_PROMPT,\n MAP_IMAGE_VALIDATOR_PROMPT,\n MAP_VALIDATOR_PROMPT,\n RETRIEVE_VALIDATOR_PROMPT,\n)\n\n__all__ = [\n # agent prompts\n \"CODE_AGENT_SYSTEM_PROMPT\",\n \"DATA_DISCOVERY_AGENT_INITIAL_PLAN_PROMPT\",\n \"DATA_DISCOVERY_AGENT_REPORT_PROMPT\",\n \"DATA_DISCOVERY_AGENT_TASK_PROMPT\",\n \"DATA_DISCOVERY_AGENT_UPDATE_PLAN_POST_MESSAGES_PROMPT\",\n \"DATA_DISCOVERY_AGENT_UPDATE_PLAN_PRE_MESSAGES_PROMPT\",\n \"FINAL_ANSWER_POST_MESSAGES_PROMPT\",\n \"FINAL_ANSWER_PRE_MESSAGES_PROMPT\",\n # context search\n \"CONTEXT_SEARCH_PROMPT\",\n # prompt cache\n \"PromptManager\",\n # prompt factory\n \"PromptFactory\",\n # utils\n \"ONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION\",\n \"ONE_TO_ONE_OUTPUT_FORMAT_INSTRUCTION\",\n # validator\n \"FLAT_MAP_IMAGE_VALIDATOR_PROMPT\",\n \"FLAT_MAP_VALIDATOR_PROMPT\",\n \"MAP_IMAGE_VALIDATOR_PROMPT\",\n \"MAP_VALIDATOR_PROMPT\",\n \"RETRIEVE_VALIDATOR_PROMPT\",\n]\n" }, { "path": "src/palimpzest/prompts/agent_prompts.py", "content": "CODE_AGENT_SYSTEM_PROMPT = \"\"\"You are an expert assistant who can solve any task using code blobs. You will be given a task to solve as best you can.\nTo do so, you have been given access to a list of tools: these tools are basically Python functions which you can call with code.\nTo solve the task, you must plan forward to proceed in a series of steps, in a cycle of 'Thought:', '', and 'Observation:' sequences.\n\nAt each step, in the 'Thought:' sequence, you should first explain your reasoning towards solving the task and the tools that you want to use.\nThen in the '' sequence, you should write the code in simple Python. The code sequence must end with '' sequence.\nDuring each intermediate step, you can use 'print()' to save whatever important information you will then need.\nThese print outputs will then appear in the 'Observation:' field, which will be available as input for the next step.\nIn the end you have to return a final answer using the `final_answer` tool.\n\nHere are a few examples using notional tools:\n---\nTask: \"Generate an image of the oldest person in this document.\"\n\nThought: I will proceed step by step and use the following tools: `document_qa` to find the oldest person in the document, then `image_generator` to generate an image according to the answer.\n\nanswer = document_qa(document=document, question=\"Who is the oldest person mentioned?\")\nprint(answer)\n\nObservation: \"The oldest person in the document is John Doe, a 55 year old lumberjack living in Newfoundland.\"\n\nThought: I will now generate an image showcasing the oldest person.\n\nimage = image_generator(\"A portrait of John Doe, a 55-year-old man living in Canada.\")\nfinal_answer(image)\n\n\n---\nTask: \"What is the result of the following operation: 5 + 3 + 1294.678?\"\n\nThought: I will use python code to compute the result of the operation and then return the final answer using the `final_answer` tool\n\nresult = 5 + 3 + 1294.678\nfinal_answer(result)\n\n\n---\nTask:\n\"Answer the question in the variable `question` about the image stored in the variable `image`. The question is in French.\nYou have been provided with these additional arguments, that you can access using the keys as variables in your python code:\n{'question': 'Quel est l'animal sur l'image?', 'image': 'path/to/image.jpg'}\"\n\nThought: I will use the following tools: `translator` to translate the question into English and then `image_qa` to answer the question on the input image.\n\ntranslated_question = translator(question=question, src_lang=\"French\", tgt_lang=\"English\")\nprint(f\"The translated question is {translated_question}.\")\nanswer = image_qa(image=image, question=translated_question)\nfinal_answer(f\"The answer is {answer}\")\n\n\n---\nTask:\nIn a 1979 interview, Stanislaus Ulam discusses with Martin Sherwin about other great physicists of his time, including Oppenheimer.\nWhat does he say was the consequence of Einstein learning too much math on his creativity, in one word?\n\nThought: I need to find and read the 1979 interview of Stanislaus Ulam with Martin Sherwin.\n\npages = web_search(query=\"1979 interview Stanislaus Ulam Martin Sherwin physicists Einstein\")\nprint(pages)\n\nObservation:\nNo result found for query \"1979 interview Stanislaus Ulam Martin Sherwin physicists Einstein\".\n\nThought: The query was maybe too restrictive and did not find any results. Let's try again with a broader query.\n\npages = web_search(query=\"1979 interview Stanislaus Ulam\")\nprint(pages)\n\nObservation:\nFound 6 pages:\n[Stanislaus Ulam 1979 interview](https://ahf.nuclearmuseum.org/voices/oral-histories/stanislaus-ulams-interview-1979/)\n\n[Ulam discusses Manhattan Project](https://ahf.nuclearmuseum.org/manhattan-project/ulam-manhattan-project/)\n\n(truncated)\n\nThought: I will read the first 2 pages to know more.\n\nfor url in [\"https://ahf.nuclearmuseum.org/voices/oral-histories/stanislaus-ulams-interview-1979/\", \"https://ahf.nuclearmuseum.org/manhattan-project/ulam-manhattan-project/\"]:\n whole_page = visit_webpage(url)\n print(whole_page)\n print(\"\\n\" + \"=\"*80 + \"\\n\") # Print separator between pages\n\nObservation:\nManhattan Project Locations:\nLos Alamos, NM\nStanislaus Ulam was a Polish-American mathematician. He worked on the Manhattan Project at Los Alamos and later helped design the hydrogen bomb. In this interview, he discusses his work at\n(truncated)\n\nThought: I now have the final answer: from the webpages visited, Stanislaus Ulam says of Einstein: \"He learned too much mathematics and sort of diminished, it seems to me personally, it seems to me his purely physics creativity.\" Let's answer in one word.\n\nfinal_answer(\"diminished\")\n\n\n---\nTask: \"Which city has the highest population: Guangzhou or Shanghai?\"\n\nThought: I need to get the populations for both cities and compare them: I will use the tool `web_search` to get the population of both cities.\n\nfor city in [\"Guangzhou\", \"Shanghai\"]:\n print(f\"Population {city}:\", web_search(f\"{city} population\")\n\nObservation:\nPopulation Guangzhou: ['Guangzhou has a population of 15 million inhabitants as of 2021.']\nPopulation Shanghai: '26 million (2019)'\n\nThought: Now I know that Shanghai has the highest population.\n\nfinal_answer(\"Shanghai\")\n\n\n---\nTask: \"What is the current age of the pope, raised to the power 0.36?\"\n\nThought: I will use the tool `wikipedia_search` to get the age of the pope, and confirm that with a web search.\n\npope_age_wiki = wikipedia_search(query=\"current pope age\")\nprint(\"Pope age as per wikipedia:\", pope_age_wiki)\npope_age_search = web_search(query=\"current pope age\")\nprint(\"Pope age as per google search:\", pope_age_search)\n\nObservation:\nPope age: \"The pope Francis is currently 88 years old.\"\n\nThought: I know that the pope is 88 years old. Let's compute the result using python code.\n\npope_current_age = 88 ** 0.36\nfinal_answer(pope_current_age)\n\n\nAbove example were using notional tools that might not exist for you. On top of performing computations in the Python code snippets that you create, you only have access to these tools, behaving like regular python functions:\n```python\n{%- for tool in tools.values() %}\ndef {{ tool.name }}({% for arg_name, arg_info in tool.inputs.items() %}{{ arg_name }}: {{ arg_info.type }}{% if not loop.last %}, {% endif %}{% endfor %}) -> {{tool.output_type}}:\n \\\"\\\"\\\"{{ tool.description }}\n\n Args:\n {%- for arg_name, arg_info in tool.inputs.items() %}\n {{ arg_name }}: {{ arg_info.description }}\n {%- endfor %}\n \\\"\\\"\\\"\n{% endfor %}\n```\n\n{%- if managed_agents and managed_agents.values() | list %}\nYou can also give tasks to team members.\nCalling a team member works similarly to calling a tool: provide the task description as the 'task' argument. Since this team member is a real human, be as detailed and verbose as necessary in your task description.\nYou can also include any relevant variables or context using the 'additional_args' argument.\nHere is a list of the team members that you can call:\n```python\n{%- for agent in managed_agents.values() %}\ndef {{ agent.name }}(task: str, additional_args: dict[str, Any]) -> str:\n \\\"\\\"\\\"{{ agent.description }}\n\n Args:\n task: Long detailed description of the task.\n additional_args: Dictionary of extra inputs to pass to the managed agent, e.g. images, dataframes, or any other contextual data it may need.\n \\\"\\\"\\\"\n{% endfor %}\n```\n{%- endif %}\n\nHere are the rules you should always follow to solve your task:\n1. Always provide a 'Thought:' sequence, and a '' sequence ending with '', else you will fail.\n2. Use only variables that you have defined!\n3. Always use the right arguments for the tools. DO NOT pass the arguments as a dict as in 'answer = wikipedia_search({'query': \"What is the place where James Bond lives?\"})', but use the arguments directly as in 'answer = wikipedia_search(query=\"What is the place where James Bond lives?\")'.\n4. Take care to not chain too many sequential tool calls in the same code block, especially when the output format is unpredictable. For instance, a call to wikipedia_search has an unpredictable return format, so do not have another tool call that depends on its output in the same block: rather output results with print() to use them in the next block.\n5. Call a tool only when needed, and never re-do a tool call that you previously did with the exact same parameters.\n6. Don't name any new variable with the same name as a tool: for instance don't name a variable 'final_answer'.\n7. Never create any notional variables in our code, as having these in your logs will derail you from the true variables.\n8. You can use imports in your code, but only from the following list of modules: {{authorized_imports}}\n9. The state persists between code executions: so if in one step you've created variables or imported modules, these will all persist.\n10. Don't give up! You're in charge of solving the task, not providing directions to solve it.\n\n{%- if custom_instructions %}\n{{custom_instructions}}\n{%- endif %}\n\nNow Begin!\"\"\"\n\n\nDATA_DISCOVERY_AGENT_INITIAL_PLAN_PROMPT = \"\"\"You are a world expert at analyzing a situation to derive facts, and plan accordingly towards solving a task.\nBelow I will present you a task. You will need to 1. build a survey of facts known or needed to solve the task, then 2. make a plan of action to solve the task.\n\n## 1. Facts survey\nYou will build a comprehensive preparatory survey of which facts we have at our disposal and which ones we still need.\nThese \"facts\" will typically be specific names, dates, values, etc. Be sure to report the facts you look up and derive, as well as the sources where you find these facts.\nYour answer should use the below headings:\n### 1.1. Facts given in the task\nList here the specific facts given in the task that could help you (there might be nothing here).\n\n### 1.2. Facts to look up\nList here any facts that we may need to look up.\nAlso list where to find each of these, for instance a website, a file... - maybe the task contains some sources that you should re-use here.\n\n### 1.3. Facts to derive\nList here anything that we want to derive from the above by logical reasoning, for instance computation or simulation.\n\n### 1.4. Fact Sources\nList the source of each fact that you have looked up or derived. The source may be a file, a database table, a web page, etc.\nBe sure that someone can use these sources to reproduce your work. \n\nDon't make any assumptions. For each item, provide a thorough reasoning. Do not add anything else on top of the four headings above.\n\n## 2. Plan\nThen for the given task, develop a step-by-step high-level plan taking into account the above inputs and list of facts.\nThis plan should involve individual tasks based on the available tools, that if executed correctly will yield the correct answer.\nDo not skip steps, do not add any superfluous steps. Only write the high-level plan, DO NOT DETAIL INDIVIDUAL TOOL CALLS.\nAfter writing the final step of the plan, write the '' tag and stop there.\n\nYou can leverage these tools, behaving like regular python functions:\n```python\n{%- for tool in tools.values() %}\ndef {{ tool.name }}({% for arg_name, arg_info in tool.inputs.items() %}{{ arg_name }}: {{ arg_info.type }}{% if not loop.last %}, {% endif %}{% endfor %}) -> {{tool.output_type}}:\n \\\"\\\"\\\"{{ tool.description }}\n\n Args:\n {%- for arg_name, arg_info in tool.inputs.items() %}\n {{ arg_name }}: {{ arg_info.description }}\n {%- endfor %}\n \\\"\\\"\\\"\n{% endfor %}\n```\n\nThe tools you have been given will provide you with access to a dataset with the following description:\n\nContext: {{context_description}}\\n\n\n{%- if managed_agents and managed_agents.values() | list %}\nYou can also give tasks to team members.\nCalling a team member works similarly to calling a tool: provide the task description as the 'task' argument. Since this team member is a real human, be as detailed and verbose as necessary in your task description.\nYou can also include any relevant variables or context using the 'additional_args' argument.\nHere is a list of the team members that you can call:\n```python\n{%- for agent in managed_agents.values() %}\ndef {{ agent.name }}(task: str, additional_args: dict[str, Any]) -> str:\n \\\"\\\"\\\"{{ agent.description }}\n\n Args:\n task: Long detailed description of the task.\n additional_args: Dictionary of extra inputs to pass to the managed agent, e.g. images, dataframes, or any other contextual data it may need.\n \\\"\\\"\\\"\n{% endfor %}\n```\n{%- endif %}\n\n---\nNow begin! Here is your task:\n```\nsearch: {{task}}\n```\nFirst in part 1, write the facts survey, then in part 2, write your plan.\n\"\"\"\n\nDATA_DISCOVERY_AGENT_UPDATE_PLAN_PRE_MESSAGES_PROMPT = \"\"\"You are a world expert at analyzing a situation, and plan accordingly towards solving a task.\nYou have been given the following task:\n```\nsearch: {{task}}\n```\nYou are working with the following dataset:\n```\ncontext: {{context_description}}\n```\n\nBelow you will find a history of attempts made to solve this task.\nYou will first have to produce a survey of known and unknown facts, then propose a step-by-step high-level plan to solve the task.\nIf the previous tries so far have met some success, your updated plan can build on these results.\nIf you are stalled, you can make a completely new plan starting from scratch.\n\nFind the task and history below:\n\"\"\"\n\nDATA_DISCOVERY_AGENT_UPDATE_PLAN_POST_MESSAGES_PROMPT = \"\"\"Now write your updated facts below, taking into account the above history:\n## 1. Updated facts survey\n### 1.1. Facts given in the task\n### 1.2. Facts that we have learned\n### 1.3. Facts still to look up\n### 1.4. Facts still to derive\n### 1.5. Fact Sources\n\nThen write a step-by-step high-level plan to solve the task above.\n## 2. Plan\n### 2. 1. ...\nEtc.\nThis plan should involve individual tasks based on the available tools, that if executed correctly will yield the correct answer.\nBeware that you have {remaining_steps} steps remaining.\nDo not skip steps, do not add any superfluous steps. Only write the high-level plan, DO NOT DETAIL INDIVIDUAL TOOL CALLS.\nAfter writing the final step of the plan, write the '' tag and stop there.\n\nYou can leverage these tools, behaving like regular python functions:\n```python\n{%- for tool in tools.values() %}\ndef {{ tool.name }}({% for arg_name, arg_info in tool.inputs.items() %}{{ arg_name }}: {{ arg_info.type }}{% if not loop.last %}, {% endif %}{% endfor %}) -> {{tool.output_type}}:\n \\\"\\\"\\\"{{ tool.description }}\n\n Args:\n {%- for arg_name, arg_info in tool.inputs.items() %}\n {{ arg_name }}: {{ arg_info.description }}\n {%- endfor %}\\\"\\\"\\\"\n{% endfor %}\n```\n\nThe tools you have been given will provide you with access to a dataset with the following description:\n\nContext: {{context_description}}\n\n{%- if managed_agents and managed_agents.values() | list %}\nYou can also give tasks to team members.\nCalling a team member works similarly to calling a tool: provide the task description as the 'task' argument. Since this team member is a real human, be as detailed and verbose as necessary in your task description.\nYou can also include any relevant variables or context using the 'additional_args' argument.\nHere is a list of the team members that you can call:\n```python\n{%- for agent in managed_agents.values() %}\ndef {{ agent.name }}(task: str, additional_args: dict[str, Any]) -> str:\n \\\"\\\"\\\"{{ agent.description }}\n\n Args:\n task: Long detailed description of the task.\n additional_args: Dictionary of extra inputs to pass to the managed agent, e.g. images, dataframes, or any other contextual data it may need.\n \\\"\\\"\\\"\n{% endfor %}\n```\n{%- endif %}\n\nNow write your updated facts survey below, then your new plan.\n\"\"\"\n\nDATA_DISCOVERY_AGENT_TASK_PROMPT = \"\"\"You're a helpful agent named '{{name}}'.\nYou have been submitted this task by your manager.\n---\nTask:\n{{task}}\n---\nYou have also been given access to tools which will help you navigate a dataset with the following description:\n---\nContext:\n{{context_description}}\n---\nYou're helping your manager solve a search task: so make sure to not provide a one-line answer, but give as much information as possible to give them a clear understanding of the answer and the context in which you produced it. In particular, be sure to report what source(s) you use and what the contents of those source(s) contain, even if they are irrelevant for solving this task.\n\nYour final_answer WILL HAVE to contain these parts:\n### 1. Task outcome (short version):\n### 2. Task outcome (extremely detailed version):\n### 3. Additional context (if relevant):\n\nPut all these in your final_answer tool, everything that you do not pass as an argument to final_answer will be lost.\nAnd even if your task resolution is not successful, please return as much context as possible, so that your manager can act upon this feedback.\n\"\"\"\n\nDATA_DISCOVERY_AGENT_REPORT_PROMPT = \"\"\"Here is the final answer from your managed agent '{{name}}':\n{{final_answer}}\"\"\"\n\nFINAL_ANSWER_PRE_MESSAGES_PROMPT = \"\"\"An agent tried to answer a user query but it got stuck and failed to do so. You are tasked with providing an answer instead. Here is the agent's memory:\"\"\"\n\nFINAL_ANSWER_POST_MESSAGES_PROMPT = \"\"\"Based on the above, please provide an answer to the following user task:\n{{task}}\n\"\"\"\n" }, { "path": "src/palimpzest/prompts/aggregate_prompts.py", "content": "\"\"\"This file contains prompts for aggregation operations.\"\"\"\n\n### BASE PROMPTS ###\nAGG_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and an output field to generate. Your task is to generate a JSON object which aggregates the input and fills in the output field with the correct value.\nYou will be provided with a description of each input field and each output field. The field in the output JSON object can be derived using information from the context.\n\n{output_format_instruction} Finish your response with a newline character followed by ---\n\nAn example is shown below:\n---\nINPUT FIELDS:\n{example_input_fields}\n\nOUTPUT FIELDS:\n{example_output_fields}\n\nCONTEXT:\n{{{example_context}}}\n{{{second_example_context}}}\n{{{third_example_context}}}{image_disclaimer}{audio_disclaimer}\n\nAGGREGATION INSTRUCTION: {example_agg_instruction}\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: {example_reasoning}\n\nANSWER:\n{{{example_answer}}}\n---\n\"\"\"\n\nAGG_NO_REASONING_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and an output field to generate. Your task is to generate a JSON object which aggregates the input and fills in the output field with the correct value.\nYou will be provided with a description of each input field and each output field. The field in the output JSON object can be derived using information from the context.\n\n{output_format_instruction} Finish your response with a newline character followed by ---\n\nAn example is shown below:\n---\nINPUT FIELDS:\n{example_input_fields}\n\nOUTPUT FIELDS:\n{example_output_fields}\n\nCONTEXT:\n{{{example_context}}}\n{{{second_example_context}}}\n{{{third_example_context}}}{image_disclaimer}{audio_disclaimer}\n\nAGGREGATION INSTRUCTION: {example_agg_instruction}\n\nANSWER:\n{{{example_answer}}}\n---\n\"\"\"\n\n\nAGG_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and an output field to generate. Your task is to generate a JSON object which aggregates the input and fills in the output field with the correct value.\nYou will be provided with a description of each input field and each output field. The field in the output JSON object can be derived using information from the context.\n{desc_section}\n{output_format_instruction} Finish your response with a newline character followed by ---\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nOUTPUT FIELDS:\n{output_fields_desc}\n\nCONTEXT:\n{context}<>\n\nAGGREGATION INSTRUCTION: {agg_instruction}\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: \"\"\"\n\nAGG_NO_REASONING_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and an output field to generate. Your task is to generate a JSON object which aggregates the input and fills in the output field with the correct value.\nYou will be provided with a description of each input field and each output field. The field in the output JSON object can be derived using information from the context.\n{desc_section}\n{output_format_instruction} Finish your response with a newline character followed by ---\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nOUTPUT FIELDS:\n{output_fields_desc}\n\nCONTEXT:\n{context}<>\n\nAGGREGATION INSTRUCTION: {agg_instruction}\n\nANSWER: \"\"\"\n" }, { "path": "src/palimpzest/prompts/context_search.py", "content": "\nCONTEXT_SEARCH_PROMPT = \"\"\"You are a helpful agent whose job is to propose search queries that will assist in finding information that is relevant to performing a computation.\n\nPlease propose a concise sentence which will be used to search for relevant information for the following computation instruction.\n\nINSTRUCTION: {instruction}\n\nSEARCH QUERY: \n\"\"\"\n" }, { "path": "src/palimpzest/prompts/convert_prompts.py", "content": "\"\"\"This file contains prompts for convert operations.\"\"\"\n\n### BASE PROMPTS ###\nMAP_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a set of output fields to generate. Your task is to generate a JSON object which fills in the output fields with the correct values.\nYou will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the context.\n\n{output_format_instruction} Finish your response with a newline character followed by ---\n\nAn example is shown below:\n---\nINPUT FIELDS:\n{example_input_fields}\n\nOUTPUT FIELDS:\n{example_output_fields}\n\nCONTEXT:\n{{{example_context}}}{image_disclaimer}{audio_disclaimer}\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: {example_reasoning}\n\nANSWER:\n{{{example_answer}}}\n---\n\"\"\"\n\nMAP_NO_REASONING_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a set of output fields to generate. Your task is to generate a JSON object which fills in the output fields with the correct values.\nYou will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the context.\n\n{output_format_instruction} Finish your response with a newline character followed by ---\n\nAn example is shown below:\n---\nINPUT FIELDS:\n{example_input_fields}\n\nOUTPUT FIELDS:\n{example_output_fields}\n\nCONTEXT:\n{{{example_context}}}{image_disclaimer}{audio_disclaimer}\n\nANSWER:\n{{{example_answer}}}\n---\n\"\"\"\n\n\nMAP_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a set of output fields to generate. Your task is to generate a JSON object which fills in the output fields with the correct values.\nYou will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the context.\n{desc_section}\n{output_format_instruction} Finish your response with a newline character followed by ---\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nOUTPUT FIELDS:\n{output_fields_desc}\n\n<>CONTEXT:\n{context}<>\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: \"\"\"\n\nMAP_NO_REASONING_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a set of output fields to generate. Your task is to generate a JSON object which fills in the output fields with the correct values.\nYou will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the context.\n{desc_section}\n{output_format_instruction} Finish your response with a newline character followed by ---\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nOUTPUT FIELDS:\n{output_fields_desc}\n\n<>CONTEXT:\n{context}<>\n\nANSWER: \"\"\"\n" }, { "path": "src/palimpzest/prompts/critique_and_refine_prompts.py", "content": "\"\"\"This file contains prompts for CritiqueAndRefineConvert operations.\"\"\"\n\n### CRITIQUE PROMPT AND CRITERIA ###\nBASE_CRITIQUE_PROMPT = \"\"\"You are a helpful assistant tasked with critiquing the output of a model based on a given prompt.\nBelow is the original user prompt used to generate the output:\n\nORIGINAL PROMPT:\n<>\n\nHere is the output generated by the model:\n\nOUTPUT:\n{original_output}\n\nYour task is to critique the output based on the following:\n{critique_criteria}\n\n{finish_instruction}\n\"\"\"\n\nMAP_CRITIQUE_CRITERIA = \"\"\"1. Does the JSON object adhere to the required format? Highlight any structural issues.\n2. Are the values of the output fields accurate based on the provided context? If any fields are incorrect or missing, provide specific examples.\n3. Are there any logical errors in reasoning used to derive the output? Provide detailed feedback on potential mistakes.\n\"\"\"\nFILTER_CRITIQUE_CRITERIA = \"\"\"1. Does the output adhere to the required TRUE/FALSE format? Highlight any issues.\n2. Is the TRUE/FALSE determination accurate based on the provided context? If there is evidence for an incorrect determination, provide specific reasons why.\n3. Are there any logical errors in reasoning used to derive the TRUE/FALSE determination? Provide detailed feedback on potential mistakes.\n\"\"\"\n\nMAP_CRITIQUE_FINISH_INSTRUCTION = \"\"\"Finish your critique with actionable recommendations for improving the JSON object.\"\"\"\nFILTER_CRITIQUE_FINISH_INSTRUCTION = \"\"\"Finish your critique with actionable recommendations for improving the model's reasoning and answer.\"\"\"\n\n### REFINEMENT PROMPT AND CRITERIA ###\nBASE_REFINEMENT_PROMPT = \"\"\"You are a helpful assistant tasked with refining the output of a model based on a critique.\nBelow is the original user prompt used to generate the output:\n\nORIGINAL PROMPT:\n<>\n\nHere is the original output generated by the model:\n\nOUTPUT:\n{original_output}\n\nHere is the critique of the output:\n\nCRITIQUE:\n{critique_output}\n\nYour task is to refine the original output to address the critique. Ensure that:\n{refinement_criteria}\n\n{finish_instruction}\n\"\"\"\n\nMAP_REFINEMENT_CRITERIA = \"\"\"1. The answer adheres to the required JSON format specified in the original prompt.\n2. Correctly derives all values for the output fields based on the provided context.\n3. Resolves any logical errors identified in the critique.\n\"\"\"\nFILTER_REFINEMENT_CRITERIA = \"\"\"1. The answer adheres to the required TRUE/FALSE format specified in the original prompt.\n2. Correctly derives the final TRUE/FALSE determination based on the provided context.\n3. Resolves any logical errors identified in the critique.\n\"\"\"\n\nMAP_REFINEMENT_FINISH_INSTRUCTION = \"\"\"Return the refined JSON object as your final answer.\"\"\"\nFILTER_REFINEMENT_FINISH_INSTRUCTION = \"\"\"Return the final TRUE/FALSE answer.\"\"\"\n" }, { "path": "src/palimpzest/prompts/filter_prompts.py", "content": "\"\"\"This file contains prompts for filter operations.\"\"\"\n\n### BASE PROMPTS ###\nFILTER_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a filter condition. Output TRUE if the context satisfies the filter condition, and FALSE otherwise.\n\nRemember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by ---\n\nAn example is shown below:\n---\nINPUT FIELDS:\n{example_input_fields}\n\nFILTER CONDITION: {example_filter_condition}\n\nCONTEXT:\n{{{example_context}}}{image_disclaimer}{audio_disclaimer}\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: {example_reasoning}\n\nANSWER: TRUE\n---\n\"\"\"\n\nFILTER_NO_REASONING_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a filter condition. Output TRUE if the context satisfies the filter condition, and FALSE otherwise.\n\nRemember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by ---\n\nAn example is shown below:\n---\nINPUT FIELDS:\n{example_input_fields}\n\nFILTER CONDITION: {example_filter_condition}\n\nCONTEXT:\n{{{example_context}}}{image_disclaimer}{audio_disclaimer}\n\nANSWER: TRUE\n---\n\"\"\"\n\nFILTER_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a filter condition. Output TRUE if the context satisfies the filter condition, and FALSE otherwise.\n{desc_section}\nRemember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by ---\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nFILTER CONDITION: {filter_condition}\n\n<>CONTEXT:\n{context}<>\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: \"\"\"\n\nFILTER_NO_REASONING_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a filter condition. Output TRUE if the context satisfies the filter condition, and FALSE otherwise.\n{desc_section}\nRemember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by ---\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nFILTER CONDITION: {filter_condition}\n\n<>CONTEXT:\n{context}<>\n\nANSWER: \"\"\"\n" }, { "path": "src/palimpzest/prompts/join_prompts.py", "content": "\"\"\"This file contains prompts for join operations.\"\"\"\n\n### BASE PROMPTS ###\nJOIN_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with two data records and a join condition. Output TRUE if the two data records satisfy the join condition, and FALSE otherwise.\n\nRemember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by ---\n\nAn example is shown below:\n---\nLEFT INPUT FIELDS:\n{example_input_fields}\n\nRIGHT INPUT FIELDS:\n{right_example_input_fields}\n\nJOIN CONDITION: {example_join_condition}\n\nLEFT CONTEXT:\n{{{example_context}}}{image_disclaimer}{audio_disclaimer}\n\nRIGHT CONTEXT:\n{{{right_example_context}}}{right_image_disclaimer}{right_audio_disclaimer}\n\nLet's think step-by-step in order to evaluate the join condition.\n\nREASONING: {example_reasoning}\n\nANSWER: TRUE\n---\n\"\"\"\n\nJOIN_NO_REASONING_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with two data records and a join condition. Output TRUE if the two data records satisfy the join condition, and FALSE otherwise.\n\nRemember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by ---\n\nAn example is shown below:\n---\nLEFT INPUT FIELDS:\n{example_input_fields}\n\nRIGHT INPUT FIELDS:\n{right_example_input_fields}\n\nJOIN CONDITION: {example_join_condition}\n\nLEFT CONTEXT:\n{{{example_context}}}{image_disclaimer}{audio_disclaimer}\n\nRIGHT CONTEXT:\n{{{right_example_context}}}{right_image_disclaimer}{right_audio_disclaimer}\n\nANSWER: TRUE\n---\n\"\"\"\n\nJOIN_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with two data records and a join condition. Output TRUE if the two data records satisfy the join condition, and FALSE otherwise.\n{desc_section}\nRemember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by ---\n---\nLEFT INPUT FIELDS:\n{input_fields_desc}\n\nRIGHT INPUT FIELDS:\n{right_input_fields_desc}\n\nJOIN CONDITION: {join_condition}\n\n<>LEFT CONTEXT:\n{context}<>\n\nRIGHT CONTEXT:\n{right_context}<>\n\nLet's think step-by-step in order to evaluate the join condition.\n\nREASONING: \"\"\"\n\nJOIN_NO_REASONING_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with two data records and a join condition. Output TRUE if the two data records satisfy the join condition, and FALSE otherwise.\n{desc_section}\nRemember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by ---\n---\nLEFT INPUT FIELDS:\n{input_fields_desc}\n\nRIGHT INPUT FIELDS:\n{right_input_fields_desc}\n\nJOIN CONDITION: {join_condition}\n\n<>LEFT CONTEXT:\n{context}<>\n\nRIGHT CONTEXT:\n{right_context}<>\n\nANSWER: \"\"\"\n" }, { "path": "src/palimpzest/prompts/moa_aggregator_prompts.py", "content": "\"\"\"This file contains prompts for Mixture-of-Agents aggregator operations.\"\"\"\n\n### SYSTEM PROMPTS ###\nMAP_MOA_AGG_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to generate a JSON object.\nYou will be presented with one or more outputs produced by a set of models. Your task is to synthesize these responses into a single, high-quality JSON object which fills in the output fields with the correct values.\nIt is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect.\n\nYou will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the model responses.\n\n{output_format_instruction} Finish your response with a newline character followed by ---\n\nAn example is shown below:\n---\nMODEL RESPONSE 1: the text mentions the scientist's full name \"Augusta Ada King, Countess of Lovelace\" and states she was an English mathematician who worked on Babbage's Analytical Engine.\n\nMODEL RESPONSE 2: the text passage mentions the scientist's name as \"Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace\" and the scientist's birthday as \"December 10, 1815\". Therefore, the name of the scientist is \"Augusta Ada King\" and the birth year is 1815.\n\nINPUT FIELDS:\n- text: a text passage describing a scientist\n- birthday: the scientist's birthday\n\nOUTPUT FIELDS:\n- name: the name of the scientist\n- birth_year: the year the scientist was born\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: Looking at both model responses, they agree that the scientist's formal name is \"Augusta Ada King\". Model Response 2 correctly extracts the birth year from the birthday field as 1815. The responses are consistent and provide sufficient evidence for these values.\n\nANSWER:\n{{\n \"name\": \"Augusta Ada King\",\n \"birth_year\": 1815\n}}\n---\n\"\"\"\n\nFILTER_MOA_AGG_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to answer a TRUE/FALSE question.\nYou will be presented with one or more outputs produced by a set of models. Your task is to synthesize these responses into a single TRUE/FALSE answer.\nIt is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect.\n\nYou will also be provided with a description of each input field and the filter condition.\n\nRemember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by ---\n\nAn example is shown below:\n---\nMODEL RESPONSE 1: The context describes Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace, who is widely recognized as a foundational figure in computer science. Therefore, the answer is TRUE.\n\nMODEL RESPONSE 2: Based on the context provided, Ada Lovelace is indeed a foundational computer scientist, therefore the answer is TRUE.\n\nINPUT FIELDS:\n- text: a text passage describing a scientist\n- birthday: the scientist's birthday\n- image: an image of the scientist\n- recording: an audio recording of a newscast about the scientist's contributions to their field\n\nFILTER CONDITION: The subject of the input is a foundational computer scientist.\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: Both model responses agree that the context describes Ada Lovelace, who is widely recognized as a foundational figure in computer science. The evidence from the text passage supports this conclusion.\n\nANSWER: TRUE\n---\n\"\"\"\n\n### USER / INSTANCE-SPECIFIC PROMPTS ###\nMAP_MOA_AGG_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to generate a JSON object.\nYou will be presented with one or more outputs produced by a set of models. Your task is to synthesize these responses into a single, high-quality JSON object which fills in the output fields with the correct values.\nIt is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect.\n\nYou will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the model responses.\n\n{output_format_instruction} Finish your response with a newline character followed by ---\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nOUTPUT FIELDS:\n{output_fields_desc}\n\n<>{model_responses}\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: \"\"\"\n\nFILTER_MOA_AGG_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to answer a TRUE/FALSE question.\nYou will be presented with one or more outputs produced by a set of models. Your task is to synthesize these responses into a single TRUE/FALSE answer.\nIt is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect.\n\nYou will also be provided with a description of each input field and the filter condition.\n\nRemember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by ---\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nFILTER CONDITION: {filter_condition}\n\n<>{model_responses}\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: \"\"\"\n" }, { "path": "src/palimpzest/prompts/moa_proposer_prompts.py", "content": "\"\"\"This file contains prompts for MixtureOfAgentsConvert operations.\"\"\"\n\n### SYSTEM PROMPTS ###\nMAP_MOA_PROPOSER_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a set of output fields to generate. Your task is to generate a detailed and succinct analysis describing what you believe is the correct value for each output field.\nBe sure to cite information from the context as evidence of why your answers are correct. Do not hallucinate evidence.\n\nYou will be provided with a description of each input field and each output field.\n\nAn example is shown below:\n---\nINPUT FIELDS:\n{example_input_fields}\n\nOUTPUT FIELDS:\n{example_output_fields}\n\nCONTEXT:\n{{{example_context}}}{image_disclaimer}{audio_disclaimer}\n\nLet's think step-by-step in order to answer the question.\n\nANSWER: {example_answer}\n---\n\"\"\"\n\nFILTER_MOA_PROPOSER_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a filter condition. Your task is to generate a detailed and succinct analysis describing whether you believe the input satisfies the filter condition.\nBe sure to cite information from the context as evidence of why your determination is correct. Do not hallucinate evidence.\n\nYou will be provided with a description of each input field.\n\nAn example is shown below:\n---\nINPUT FIELDS:\n{example_input_fields}\n\nFILTER CONDITION: {example_filter_condition}\n\nCONTEXT:\n{{{example_context}}}{image_disclaimer}{audio_disclaimer}\n\nLet's think step-by-step in order to answer the question.\n\nANSWER: {example_answer}\n---\n\"\"\"\n\n### USER / INSTANCE-SPECIFIC PROMPTS ###\nMAP_MOA_PROPOSER_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a set of output fields to generate. Your task is to generate a detailed and succinct analysis describing what you believe is the correct value for each output field.\nBe sure to cite information from the context as evidence of why your answers are correct. Do not hallucinate evidence.\n{desc_section}\nYou will be provided with a description of each input field and each output field.\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nOUTPUT FIELDS:\n{output_fields_desc}\n\n<>CONTEXT:\n{context}<>\n\nLet's think step-by-step in order to answer the question.\n\nANSWER: \"\"\"\n\nFILTER_MOA_PROPOSER_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a filter condition. Your task is to generate a detailed and succinct analysis describing whether you believe the input satisfies the filter condition.\nBe sure to cite information from the context as evidence of why your determination is correct. Do not hallucinate evidence.\n{desc_section}\nYou will be provided with a description of each input field.\n\nAn example is shown below:\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nFILTER CONDITION: {filter_condition}\n\n<>CONTEXT:\n{context}<>\n\nLet's think step-by-step in order to answer the question.\n\nANSWER: \"\"\"\n" }, { "path": "src/palimpzest/prompts/prompt_factory.py", "content": "\"\"\"This file contains factory methods which return template prompts and return messages for chat payloads.\"\"\"\n\nimport base64\nimport json\nimport os\nfrom typing import Any\n\nfrom pydantic import BaseModel\n\nfrom palimpzest.constants import (\n LLAMA_CONTEXT_TOKENS_LIMIT,\n TOKENS_PER_CHARACTER,\n Cardinality,\n Modality,\n Model,\n PromptStrategy,\n)\nfrom palimpzest.core.elements.records import DataRecord\nfrom palimpzest.core.lib.schemas import (\n AUDIO_FIELD_TYPES,\n IMAGE_FIELD_TYPES,\n AudioBase64,\n AudioFilepath,\n ImageBase64,\n ImageFilepath,\n ImageURL,\n)\nfrom palimpzest.prompts.aggregate_prompts import (\n AGG_BASE_SYSTEM_PROMPT,\n AGG_BASE_USER_PROMPT,\n AGG_NO_REASONING_BASE_SYSTEM_PROMPT,\n AGG_NO_REASONING_BASE_USER_PROMPT,\n)\nfrom palimpzest.prompts.convert_prompts import (\n MAP_BASE_SYSTEM_PROMPT,\n MAP_BASE_USER_PROMPT,\n MAP_NO_REASONING_BASE_SYSTEM_PROMPT,\n MAP_NO_REASONING_BASE_USER_PROMPT,\n)\nfrom palimpzest.prompts.critique_and_refine_prompts import (\n BASE_CRITIQUE_PROMPT,\n BASE_REFINEMENT_PROMPT,\n FILTER_CRITIQUE_CRITERIA,\n FILTER_CRITIQUE_FINISH_INSTRUCTION,\n FILTER_REFINEMENT_CRITERIA,\n FILTER_REFINEMENT_FINISH_INSTRUCTION,\n MAP_CRITIQUE_CRITERIA,\n MAP_CRITIQUE_FINISH_INSTRUCTION,\n MAP_REFINEMENT_CRITERIA,\n MAP_REFINEMENT_FINISH_INSTRUCTION,\n)\nfrom palimpzest.prompts.filter_prompts import (\n FILTER_BASE_SYSTEM_PROMPT,\n FILTER_BASE_USER_PROMPT,\n FILTER_NO_REASONING_BASE_SYSTEM_PROMPT,\n FILTER_NO_REASONING_BASE_USER_PROMPT,\n)\nfrom palimpzest.prompts.join_prompts import (\n JOIN_BASE_SYSTEM_PROMPT,\n JOIN_BASE_USER_PROMPT,\n JOIN_NO_REASONING_BASE_SYSTEM_PROMPT,\n JOIN_NO_REASONING_BASE_USER_PROMPT,\n)\nfrom palimpzest.prompts.moa_aggregator_prompts import (\n FILTER_MOA_AGG_BASE_SYSTEM_PROMPT,\n FILTER_MOA_AGG_BASE_USER_PROMPT,\n MAP_MOA_AGG_BASE_SYSTEM_PROMPT,\n MAP_MOA_AGG_BASE_USER_PROMPT,\n)\nfrom palimpzest.prompts.moa_proposer_prompts import (\n FILTER_MOA_PROPOSER_BASE_SYSTEM_PROMPT,\n FILTER_MOA_PROPOSER_BASE_USER_PROMPT,\n MAP_MOA_PROPOSER_BASE_SYSTEM_PROMPT,\n MAP_MOA_PROPOSER_BASE_USER_PROMPT,\n)\nfrom palimpzest.prompts.split_merge_prompts import (\n FILTER_SPLIT_MERGER_BASE_SYSTEM_PROMPT,\n FILTER_SPLIT_MERGER_BASE_USER_PROMPT,\n MAP_SPLIT_MERGER_BASE_SYSTEM_PROMPT,\n MAP_SPLIT_MERGER_BASE_USER_PROMPT,\n)\nfrom palimpzest.prompts.split_proposer_prompts import (\n FILTER_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT,\n FILTER_SPLIT_PROPOSER_BASE_USER_PROMPT,\n MAP_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT,\n MAP_SPLIT_PROPOSER_BASE_USER_PROMPT,\n)\nfrom palimpzest.prompts.utils import (\n AGG_AUDIO_DISCLAIMER,\n AGG_EXAMPLE_ANSWER,\n AGG_EXAMPLE_OUTPUT_FIELDS,\n AGG_EXAMPLE_REASONING,\n AGG_IMAGE_DISCLAIMER,\n AGG_JOB_INSTRUCTION,\n AUDIO_DISCLAIMER,\n AUDIO_EXAMPLE_ANSWER,\n AUDIO_EXAMPLE_CONTEXT,\n AUDIO_EXAMPLE_INPUT_FIELDS,\n AUDIO_EXAMPLE_OUTPUT_FIELDS,\n AUDIO_EXAMPLE_REASONING,\n AUDIO_SENTENCE_EXAMPLE_ANSWER,\n DESC_SECTION,\n EXAMPLE_AGG_INSTRUCTION,\n EXAMPLE_FILTER_CONDITION,\n EXAMPLE_JOIN_CONDITION,\n FILTER_EXAMPLE_REASONING,\n FILTER_JOB_INSTRUCTION,\n IMAGE_DISCLAIMER,\n IMAGE_EXAMPLE_ANSWER,\n IMAGE_EXAMPLE_CONTEXT,\n IMAGE_EXAMPLE_INPUT_FIELDS,\n IMAGE_EXAMPLE_OUTPUT_FIELDS,\n IMAGE_EXAMPLE_REASONING,\n IMAGE_SENTENCE_EXAMPLE_ANSWER,\n JOIN_EXAMPLE_REASONING,\n JOIN_JOB_INSTRUCTION,\n MAP_JOB_INSTRUCTION,\n ONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION,\n ONE_TO_ONE_OUTPUT_FORMAT_INSTRUCTION,\n PROPOSER_JOB_INSTRUCTION,\n RIGHT_AUDIO_DISCLAIMER,\n RIGHT_AUDIO_EXAMPLE_CONTEXT,\n RIGHT_AUDIO_EXAMPLE_INPUT_FIELDS,\n RIGHT_IMAGE_DISCLAIMER,\n RIGHT_IMAGE_EXAMPLE_CONTEXT,\n RIGHT_IMAGE_EXAMPLE_INPUT_FIELDS,\n RIGHT_TEXT_EXAMPLE_CONTEXT,\n RIGHT_TEXT_EXAMPLE_INPUT_FIELDS,\n SECOND_AUDIO_EXAMPLE_CONTEXT,\n SECOND_IMAGE_EXAMPLE_CONTEXT,\n SECOND_TEXT_EXAMPLE_CONTEXT,\n TEXT_EXAMPLE_ANSWER,\n TEXT_EXAMPLE_CONTEXT,\n TEXT_EXAMPLE_INPUT_FIELDS,\n TEXT_EXAMPLE_OUTPUT_FIELDS,\n TEXT_EXAMPLE_REASONING,\n TEXT_SENTENCE_EXAMPLE_ANSWER,\n THIRD_AUDIO_EXAMPLE_CONTEXT,\n THIRD_IMAGE_EXAMPLE_CONTEXT,\n THIRD_TEXT_EXAMPLE_CONTEXT,\n)\n\n\ndef _detect_image_media_type(filepath: str | None = None, base64_data: str | None = None) -> str:\n \"\"\"Detect image media type from file extension or base64 magic bytes.\"\"\"\n if filepath:\n ext = os.path.splitext(filepath)[1].lower()\n ext_map = {\".png\": \"image/png\", \".jpg\": \"image/jpeg\", \".jpeg\": \"image/jpeg\",\n \".gif\": \"image/gif\", \".webp\": \"image/webp\"}\n if ext in ext_map:\n return ext_map[ext]\n\n if base64_data:\n try:\n header = base64.b64decode(base64_data[:32])\n if header[:8] == b\"\\x89PNG\\r\\n\\x1a\\n\":\n return \"image/png\"\n if header[:3] == b\"\\xff\\xd8\\xff\":\n return \"image/jpeg\"\n if header[:6] in (b\"GIF87a\", b\"GIF89a\"):\n return \"image/gif\"\n if header[:4] == b\"RIFF\" and header[8:12] == b\"WEBP\":\n return \"image/webp\"\n except Exception:\n pass\n\n return \"image/jpeg\"\n\n\nclass PromptFactory:\n \"\"\"Factory class for generating prompts for the Generator given the input(s).\"\"\"\n\n BASE_SYSTEM_PROMPT_MAP = {\n # agg user prompts\n PromptStrategy.AGG: AGG_BASE_SYSTEM_PROMPT,\n PromptStrategy.AGG_NO_REASONING: AGG_NO_REASONING_BASE_SYSTEM_PROMPT,\n\n # filter system prompts\n PromptStrategy.FILTER: FILTER_BASE_SYSTEM_PROMPT,\n PromptStrategy.FILTER_NO_REASONING: FILTER_NO_REASONING_BASE_SYSTEM_PROMPT,\n PromptStrategy.FILTER_CRITIC: None,\n PromptStrategy.FILTER_REFINE: None,\n PromptStrategy.FILTER_MOA_PROPOSER: FILTER_MOA_PROPOSER_BASE_SYSTEM_PROMPT,\n PromptStrategy.FILTER_MOA_AGG: FILTER_MOA_AGG_BASE_SYSTEM_PROMPT,\n PromptStrategy.FILTER_SPLIT_PROPOSER: FILTER_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT,\n PromptStrategy.FILTER_SPLIT_MERGER: FILTER_SPLIT_MERGER_BASE_SYSTEM_PROMPT,\n\n # join system prompts\n PromptStrategy.JOIN: JOIN_BASE_SYSTEM_PROMPT,\n PromptStrategy.JOIN_NO_REASONING: JOIN_NO_REASONING_BASE_SYSTEM_PROMPT,\n\n # map system prompts\n PromptStrategy.MAP: MAP_BASE_SYSTEM_PROMPT,\n PromptStrategy.MAP_NO_REASONING: MAP_NO_REASONING_BASE_SYSTEM_PROMPT,\n PromptStrategy.MAP_CRITIC: None,\n PromptStrategy.MAP_REFINE: None,\n PromptStrategy.MAP_MOA_PROPOSER: MAP_MOA_PROPOSER_BASE_SYSTEM_PROMPT,\n PromptStrategy.MAP_MOA_AGG: MAP_MOA_AGG_BASE_SYSTEM_PROMPT,\n PromptStrategy.MAP_SPLIT_PROPOSER: MAP_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT,\n PromptStrategy.MAP_SPLIT_MERGER: MAP_SPLIT_MERGER_BASE_SYSTEM_PROMPT,\n }\n BASE_USER_PROMPT_MAP = {\n # agg user prompts\n PromptStrategy.AGG: AGG_BASE_USER_PROMPT,\n PromptStrategy.AGG_NO_REASONING: AGG_NO_REASONING_BASE_USER_PROMPT,\n\n # filter user prompts\n PromptStrategy.FILTER: FILTER_BASE_USER_PROMPT,\n PromptStrategy.FILTER_NO_REASONING: FILTER_NO_REASONING_BASE_USER_PROMPT,\n PromptStrategy.FILTER_CRITIC: BASE_CRITIQUE_PROMPT,\n PromptStrategy.FILTER_REFINE: BASE_REFINEMENT_PROMPT,\n PromptStrategy.FILTER_MOA_PROPOSER: FILTER_MOA_PROPOSER_BASE_USER_PROMPT,\n PromptStrategy.FILTER_MOA_AGG: FILTER_MOA_AGG_BASE_USER_PROMPT,\n PromptStrategy.FILTER_SPLIT_PROPOSER: FILTER_SPLIT_PROPOSER_BASE_USER_PROMPT,\n PromptStrategy.FILTER_SPLIT_MERGER: FILTER_SPLIT_MERGER_BASE_USER_PROMPT,\n\n # join user prompts\n PromptStrategy.JOIN: JOIN_BASE_USER_PROMPT,\n PromptStrategy.JOIN_NO_REASONING: JOIN_NO_REASONING_BASE_USER_PROMPT,\n\n # map user prompts\n PromptStrategy.MAP: MAP_BASE_USER_PROMPT,\n PromptStrategy.MAP_NO_REASONING: MAP_NO_REASONING_BASE_USER_PROMPT,\n PromptStrategy.MAP_CRITIC: BASE_CRITIQUE_PROMPT,\n PromptStrategy.MAP_REFINE: BASE_REFINEMENT_PROMPT,\n PromptStrategy.MAP_MOA_PROPOSER: MAP_MOA_PROPOSER_BASE_USER_PROMPT,\n PromptStrategy.MAP_MOA_AGG: MAP_MOA_AGG_BASE_USER_PROMPT,\n PromptStrategy.MAP_SPLIT_PROPOSER: MAP_SPLIT_PROPOSER_BASE_USER_PROMPT,\n PromptStrategy.MAP_SPLIT_MERGER: MAP_SPLIT_MERGER_BASE_USER_PROMPT,\n }\n\n def __init__(self, prompt_strategy: PromptStrategy, model: Model, cardinality: Cardinality, desc: str | None = None) -> None:\n self.prompt_strategy = prompt_strategy\n self.model = model\n self.cardinality = cardinality\n self.desc = desc\n\n def _get_context(self, candidate: DataRecord | list[DataRecord], input_fields: list[str]) -> str:\n \"\"\"\n Returns the context for the prompt.\n\n Args:\n candidate (DataRecord): The input record.\n input_fields (list[str]): The input fields.\n\n Returns:\n str: The context.\n \"\"\"\n # TODO: remove mask_filepaths=True after SemBench evaluation\n # get context from input record (project_cols will be None if not provided in kwargs)\n if isinstance(candidate, list):\n context: list[dict] = [record.to_dict(include_bytes=False, project_cols=input_fields, mask_filepaths=True) for record in candidate]\n else:\n context: dict = candidate.to_dict(include_bytes=False, project_cols=input_fields, mask_filepaths=True)\n\n # TODO: MOVE THIS LOGIC INTO A CHUNKING / CONTEXT MANAGEMENT CLASS\n # - this class should be able to:\n # - handle the context length of different models (i.e. self.model should be an input)\n # - handle images\n # - handle the issue with `original_messages` (ask Matt if this is not clear)\n # TODO: this does not work for image prompts\n # TODO: this ignores the size of the `orignal_messages` in critique and refine prompts\n # NOTE: llama models are disallowed for aggregation so we can assume context is a dict here\n # cut down on context based on window length\n if self.model.is_llama_model():\n assert isinstance(context, dict), \"Llama models are not allowed for aggregation operations.\"\n total_context_len = len(json.dumps(context, indent=2))\n\n # sort fields by length and progressively strip from the longest field until it is short enough;\n # NOTE: LLAMA_CONTEXT_TOKENS_LIMIT is a rough estimate which leaves room for the rest of the prompt text\n while total_context_len * TOKENS_PER_CHARACTER > LLAMA_CONTEXT_TOKENS_LIMIT:\n # sort fields by length\n field_lengths = [(field, len(value) if value is not None else 0) for field, value in context.items()]\n sorted_fields = sorted(field_lengths, key=lambda item: item[1], reverse=True)\n\n # get field with longest context\n longest_field_name, longest_field_length = sorted_fields[0]\n\n # trim the field\n context_factor = LLAMA_CONTEXT_TOKENS_LIMIT / (total_context_len * TOKENS_PER_CHARACTER)\n keep_frac_idx = int(longest_field_length * context_factor)\n context[longest_field_name] = context[longest_field_name][:keep_frac_idx]\n\n # update total context length\n total_context_len = len(json.dumps(context, indent=2))\n\n return json.dumps(context, indent=2)\n\n def _get_input_fields(self, candidate: DataRecord, **kwargs) -> list[str]:\n \"\"\"\n The list of input fields to be templated into the prompt(s).\n If the user provides a list of \"project_cols\" in kwargs, then this list will be returned.\n Otherwise, this function returns the list of all field names in the candidate record.\n\n Args:\n candidate (DataRecord): The input record.\n kwargs: The keyword arguments provided by the user.\n\n Returns:\n list[str]: The list of input field names.\n \"\"\"\n # NOTE: joins will include left and right input fields in project_cols, so we have to check\n # if the field is in the candidate record\n input_fields = kwargs.get(\"project_cols\", candidate.get_field_names())\n input_fields = [field for field in input_fields if field in candidate.get_field_names()]\n return input_fields\n\n def _get_input_modalities(self, candidate: DataRecord, input_fields: list[str]) -> set[Modality]:\n \"\"\"\n The list of input modalities for the given input fields.\n\n Args:\n candidate (DataRecord): The input record.\n input_fields (list[str]): The input fields.\n\n Returns:\n set[Modality]: The list of input modalities.\n \"\"\"\n input_modalities = []\n for field_name in input_fields:\n field_type = candidate.get_field_type(field_name)\n if field_type.annotation in IMAGE_FIELD_TYPES:\n input_modalities.append(Modality.IMAGE)\n elif field_type.annotation in AUDIO_FIELD_TYPES:\n input_modalities.append(Modality.AUDIO)\n else:\n input_modalities.append(Modality.TEXT)\n\n return set(input_modalities)\n\n def _get_modalities_str(self, input_modalities: set[Modality]) -> str:\n \"\"\"\n Returns a format string to reflect the input modalities.\n\n Args:\n input_modalities (set[Modality]): The input modalities.\n\n Returns:\n str: The string to reflect the input modalities.\n \"\"\"\n if input_modalities == {Modality.TEXT}:\n return \"text\"\n elif input_modalities == {Modality.IMAGE}:\n return \"image(s)\"\n elif input_modalities == {Modality.AUDIO}:\n return \"audio\"\n elif input_modalities == {Modality.TEXT, Modality.IMAGE}:\n return \"text and/or image(s)\"\n elif input_modalities == {Modality.TEXT, Modality.AUDIO}:\n return \"text and/or audio\"\n elif input_modalities == {Modality.IMAGE, Modality.AUDIO}:\n return \"image(s) and/or audio\"\n elif input_modalities == {Modality.TEXT, Modality.IMAGE, Modality.AUDIO}:\n return \"text, image(s), and/or audio\"\n\n def _get_input_fields_desc(self, candidate: DataRecord, input_fields: list[str]) -> str:\n \"\"\"\n Returns a multi-line description of each input field for the prompt.\n\n Args:\n input_fields (list[str]): The input fields.\n\n Returns:\n str: The input fields description.\n \"\"\"\n input_fields_desc = \"\"\n for field_name in input_fields:\n input_fields_desc += f\"- {field_name}: {candidate.get_field_type(field_name).description}\\n\"\n\n return input_fields_desc[:-1]\n\n def _get_output_fields_desc(self, output_fields: list[str], **kwargs) -> str:\n \"\"\"\n Returns a multi-line description of each output field for the prompt.\n\n Args:\n output_fields (list[str]): The output fields.\n kwargs: The keyword arguments provided by the user.\n\n Returns:\n str: The output fields description.\n \"\"\"\n output_fields_desc = \"\"\n output_schema: type[BaseModel] = kwargs.get(\"output_schema\")\n if self.prompt_strategy.is_map_prompt() or self.prompt_strategy.is_agg_prompt():\n assert output_schema is not None, \"Output schema must be provided for convert prompts.\"\n\n for field_name in sorted(output_fields):\n desc = output_schema.model_fields[field_name].description\n output_fields_desc += f\"- {field_name}: {'no description available' if desc is None else desc}\\n\"\n\n # strip the last newline characters from the field descriptions and return\n return output_fields_desc[:-1]\n\n def _get_agg_instruction(self, **kwargs) -> str | None:\n \"\"\"\n Returns the aggregation instruction for the aggregation operation.\n\n Returns:\n str | None: The aggregation instruction (if applicable).\n \"\"\"\n agg_instruction = kwargs.get(\"agg_instruction\")\n if self.prompt_strategy.is_agg_prompt():\n assert agg_instruction is not None, \"Aggregation instruction must be provided for aggregation operations.\"\n\n return agg_instruction\n\n def _get_filter_condition(self, **kwargs) -> str | None:\n \"\"\"\n Returns the filter condition for the filter operation.\n\n Returns:\n str | None: The filter condition (if applicable).\n \"\"\"\n filter_condition = kwargs.get(\"filter_condition\")\n if self.prompt_strategy.is_filter_prompt():\n assert filter_condition is not None, \"Filter condition must be provided for filter operations.\"\n\n return filter_condition\n\n def _get_join_condition(self, **kwargs) -> str | None:\n \"\"\"\n Returns the join condition for the join operation.\n\n Returns:\n str | None: The join condition (if applicable).\n \"\"\"\n join_condition = kwargs.get(\"join_condition\")\n if self.prompt_strategy.is_join_prompt():\n assert join_condition is not None, \"Join condition must be provided for join operations.\"\n\n return join_condition\n\n def _get_original_output(self, **kwargs) -> str | None:\n \"\"\"\n Returns the original output from a previous model generation for the critique and refinement operations.\n\n Args:\n kwargs: The keyword arguments provided by the user.\n\n Returns:\n str | None: The original output.\n \"\"\"\n original_output = kwargs.get(\"original_output\")\n if self.prompt_strategy.is_critic_prompt() or self.prompt_strategy.is_refine_prompt():\n assert original_output is not None, (\n \"Original output must be provided for critique and refinement operations.\"\n )\n\n return original_output\n\n def _get_critique_output(self, **kwargs) -> str | None:\n \"\"\"\n Returns the critique output for the refinement operation.\n\n Args:\n kwargs: The keyword arguments provided by the user.\n\n Returns:\n str | None: The critique output.\n \"\"\"\n critique_output = kwargs.get(\"critique_output\")\n if self.prompt_strategy.is_refine_prompt():\n assert critique_output is not None, \"Critique output must be provided for refinement operations.\"\n\n return critique_output\n\n def _get_model_responses(self, **kwargs) -> str | None:\n \"\"\"\n Returns the model responses for the mixture-of-agents aggregation operation.\n\n Args:\n kwargs: The keyword arguments provided by the user.\n\n Returns:\n str | None: The model responses.\n \"\"\"\n model_responses = None\n if self.prompt_strategy.is_moa_aggregator_prompt():\n model_responses = \"\"\n for idx, model_response in enumerate(kwargs.get(\"model_responses\")):\n model_responses += f\"MODEL RESPONSE {idx + 1}: {model_response.rstrip()}\\n\\n\"\n model_responses = model_responses.rstrip() if model_responses is not None else None\n\n return model_responses\n\n def _get_chunk_outputs(self, **kwargs) -> str | None:\n \"\"\"\n Returns the chunk outputs for the split-convert.\n\n Args:\n kwargs: The keyword arguments provided by the user.\n\n Returns:\n str | None: The chunk outputs.\n \"\"\"\n chunk_outputs = None\n if self.prompt_strategy.is_split_merger_prompt():\n chunk_outputs = \"\"\n for idx, chunk_output in enumerate(kwargs.get(\"chunk_outputs\")):\n chunk_outputs += f\"CHUNK OUTPUT {idx + 1}: {chunk_output.rstrip()}\\n\\n\"\n chunk_outputs = chunk_outputs.rstrip() if chunk_outputs is not None else None\n\n return chunk_outputs\n\n def _get_output_format_instruction(self) -> str:\n \"\"\"\n Returns the output format instruction based on the cardinality.\n\n Returns:\n str: The output format instruction.\n \"\"\"\n return (\n ONE_TO_ONE_OUTPUT_FORMAT_INSTRUCTION\n if self.cardinality == Cardinality.ONE_TO_ONE\n else ONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION\n )\n\n def _get_job_instruction(self, input_modalities: set[Modality]) -> str | None:\n \"\"\"\n Returns the job instruction based on the prompt strategy.\n\n Args:\n input_modalities (set[Modality]): The modalities of the input fields.\n\n Returns:\n str | None: The job instruction.\n \"\"\"\n # get the job instruction based on the prompt strategy\n job_instruction = None\n if self.prompt_strategy.is_moa_proposer_prompt() or self.prompt_strategy.is_split_proposer_prompt():\n job_instruction = PROPOSER_JOB_INSTRUCTION\n elif self.prompt_strategy.is_map_prompt():\n job_instruction = MAP_JOB_INSTRUCTION\n elif self.prompt_strategy.is_filter_prompt():\n job_instruction = FILTER_JOB_INSTRUCTION\n elif self.prompt_strategy.is_join_prompt():\n job_instruction = JOIN_JOB_INSTRUCTION\n elif self.prompt_strategy.is_agg_prompt():\n job_instruction = AGG_JOB_INSTRUCTION\n\n # format the job instruction based on the input modalities\n modalities = self._get_modalities_str(input_modalities)\n if job_instruction is not None:\n job_instruction = job_instruction.format(modalities=modalities)\n\n return job_instruction\n\n def _get_desc_section(self) -> str:\n \"\"\"\n Returns the description section for the prompt.\n\n Returns:\n str: The description section (if applicable).\n \"\"\"\n desc_section = \"\"\n if self.desc is not None:\n desc_section = DESC_SECTION.format(desc=self.desc)\n\n return desc_section\n\n def _get_critique_criteria(self) -> str | None:\n \"\"\"\n Returns the critique criteria for the critique operation.\n\n Returns:\n str | None: The critique criteria (if applicable).\n \"\"\"\n critique_criteria = None\n if self.prompt_strategy.is_critic_prompt():\n critique_criteria = MAP_CRITIQUE_CRITERIA if self.prompt_strategy.is_map_prompt() else FILTER_CRITIQUE_CRITERIA\n\n return critique_criteria\n\n def _get_refinement_criteria(self) -> str | None:\n \"\"\"\n Returns the refinement criteria for the refinement operation.\n\n Returns:\n str | None: The refinement criteria (if applicable).\n \"\"\"\n refinement_criteria = None\n if self.prompt_strategy.is_refine_prompt():\n refinement_criteria = MAP_REFINEMENT_CRITERIA if self.prompt_strategy.is_map_prompt() else FILTER_REFINEMENT_CRITERIA\n\n return refinement_criteria\n\n def _get_finish_instruction(self) -> str | None:\n \"\"\"\n Returns the finish instruction for the critique and refinement operations.\n\n Returns:\n str | None: The finish instruction (if applicable).\n \"\"\"\n finish_instruction = None\n if self.prompt_strategy.is_critic_prompt():\n finish_instruction = MAP_CRITIQUE_FINISH_INSTRUCTION if self.prompt_strategy.is_map_prompt() else FILTER_CRITIQUE_FINISH_INSTRUCTION\n elif self.prompt_strategy.is_refine_prompt():\n finish_instruction = MAP_REFINEMENT_FINISH_INSTRUCTION if self.prompt_strategy.is_map_prompt() else FILTER_REFINEMENT_FINISH_INSTRUCTION\n\n return finish_instruction\n\n def _get_example_input_fields(self, input_modalities: set[Modality], right: bool = False) -> str:\n \"\"\"\n Returns the example input fields for the prompt.\n\n Args:\n input_modalities (set[Modality]): The modalities of the input fields.\n right (bool): Whether to return the right input fields for the join prompt.\n\n Returns:\n str: The example input fields.\n \"\"\"\n input_modality_to_example_input_fields = {\n Modality.TEXT: RIGHT_TEXT_EXAMPLE_INPUT_FIELDS if right else TEXT_EXAMPLE_INPUT_FIELDS,\n Modality.IMAGE: RIGHT_IMAGE_EXAMPLE_INPUT_FIELDS if right else IMAGE_EXAMPLE_INPUT_FIELDS,\n Modality.AUDIO: RIGHT_AUDIO_EXAMPLE_INPUT_FIELDS if right else AUDIO_EXAMPLE_INPUT_FIELDS,\n }\n\n example_input_fields = \"\"\n for input_modality in input_modalities:\n example_input_fields += input_modality_to_example_input_fields[input_modality].rstrip()\n example_input_fields = example_input_fields.lstrip() + \"\\n\"\n\n return example_input_fields\n\n def _get_example_output_fields(self, input_modalities: set[Modality]) -> str:\n \"\"\"\n Returns the example output fields for the prompt.\n\n Returns:\n str: The example output fields.\n \"\"\"\n if self.prompt_strategy.is_agg_prompt():\n return AGG_EXAMPLE_OUTPUT_FIELDS\n\n input_modality_to_example_output_fields = {\n Modality.TEXT: TEXT_EXAMPLE_OUTPUT_FIELDS,\n Modality.IMAGE: IMAGE_EXAMPLE_OUTPUT_FIELDS,\n Modality.AUDIO: AUDIO_EXAMPLE_OUTPUT_FIELDS,\n }\n\n example_output_fields = \"\"\n for input_modality in input_modalities:\n example_output_fields += input_modality_to_example_output_fields[input_modality].rstrip()\n example_output_fields = example_output_fields.lstrip() + \"\\n\"\n\n return example_output_fields\n\n def _get_example_context(self, input_modalities: set[Modality], right: bool = False, second: bool = False, third: bool = False) -> str:\n \"\"\"\n Returns the example context for the prompt.\n\n Returns:\n str: The example context.\n \"\"\"\n assert not (second and third), \"Cannot have both second and third example contexts.\"\n assert not (right and (second or third)), \"Right context is only used for joins; second and third contexts only use for aggregations.\"\n text_example_context = TEXT_EXAMPLE_CONTEXT\n image_example_context = IMAGE_EXAMPLE_CONTEXT\n audio_example_context = AUDIO_EXAMPLE_CONTEXT\n if second:\n text_example_context = SECOND_TEXT_EXAMPLE_CONTEXT\n image_example_context = SECOND_IMAGE_EXAMPLE_CONTEXT\n audio_example_context = SECOND_AUDIO_EXAMPLE_CONTEXT\n elif third:\n text_example_context = THIRD_TEXT_EXAMPLE_CONTEXT\n image_example_context = THIRD_IMAGE_EXAMPLE_CONTEXT\n audio_example_context = THIRD_AUDIO_EXAMPLE_CONTEXT\n\n input_modality_to_example_context = {\n Modality.TEXT: RIGHT_TEXT_EXAMPLE_CONTEXT if right else text_example_context,\n Modality.IMAGE: RIGHT_IMAGE_EXAMPLE_CONTEXT if right else image_example_context,\n Modality.AUDIO: RIGHT_AUDIO_EXAMPLE_CONTEXT if right else audio_example_context,\n }\n\n example_context = \"\"\n for input_modality in input_modalities:\n example_context += input_modality_to_example_context[input_modality].rstrip() + \",\"\n example_context = example_context[:-1] + \"\\n\"\n\n return example_context\n\n def _get_image_disclaimer(self, input_modalities: set[Modality], right: bool = False, agg: bool = False) -> str:\n \"\"\"\n Returns the image disclaimer for the prompt. The disclaimer must be an empty string\n for non-image prompts.\n\n Returns:\n str: The image disclaimer. If this is a text prompt then it is an empty string.\n \"\"\"\n assert not (right and agg), \"Right image disclaimer is only used for joins; agg image disclaimer only used for aggregations.\"\n image_disclaimer = AGG_IMAGE_DISCLAIMER if agg else IMAGE_DISCLAIMER\n image_disclaimer = RIGHT_IMAGE_DISCLAIMER if right else image_disclaimer\n return image_disclaimer if Modality.IMAGE in input_modalities else \"\"\n\n def _get_audio_disclaimer(self, input_modalities: set[Modality], right: bool = False, agg: bool = False) -> str:\n \"\"\"\n Returns the audio disclaimer for the prompt. The disclaimer must be an empty string\n for non-audio prompts.\n\n Returns:\n str: The audio disclaimer. If this is a text prompt then it is an empty string.\n \"\"\"\n assert not (right and agg), \"Right audio disclaimer is only used for joins; agg audio disclaimer only used for aggregations.\"\n audio_disclaimer = AGG_AUDIO_DISCLAIMER if agg else AUDIO_DISCLAIMER\n audio_disclaimer = RIGHT_AUDIO_DISCLAIMER if right else audio_disclaimer\n return audio_disclaimer if Modality.AUDIO in input_modalities else \"\"\n\n def _get_example_reasoning(self, input_modalities: set[Modality]) -> str:\n \"\"\"\n Returns the example reasoning for the prompt.\n\n Returns:\n str: The example reasoning.\n \"\"\"\n if self.prompt_strategy.is_filter_prompt():\n return FILTER_EXAMPLE_REASONING\n elif self.prompt_strategy.is_join_prompt():\n return JOIN_EXAMPLE_REASONING\n elif self.prompt_strategy.is_agg_prompt():\n return AGG_EXAMPLE_REASONING\n\n input_modality_to_example_reasoning = {\n Modality.TEXT: TEXT_EXAMPLE_REASONING,\n Modality.IMAGE: IMAGE_EXAMPLE_REASONING,\n Modality.AUDIO: AUDIO_EXAMPLE_REASONING,\n }\n\n example_reasoning = \"\"\n for input_modality in input_modalities:\n example_reasoning += input_modality_to_example_reasoning[input_modality] + \" \"\n example_reasoning = example_reasoning.rstrip()\n\n return example_reasoning\n\n def _get_example_answer(self, input_modalities: set[Modality]) -> str:\n \"\"\"\n Returns the example answer for the prompt.\n\n Returns:\n str: The example answer.\n \"\"\"\n if self.prompt_strategy.is_agg_prompt():\n return AGG_EXAMPLE_ANSWER\n\n use_sentence_answers = self.prompt_strategy.is_split_proposer_prompt() or self.prompt_strategy.is_moa_proposer_prompt()\n input_modality_to_example_answer = {\n Modality.TEXT: TEXT_SENTENCE_EXAMPLE_ANSWER if use_sentence_answers else TEXT_EXAMPLE_ANSWER,\n Modality.IMAGE: IMAGE_SENTENCE_EXAMPLE_ANSWER if use_sentence_answers else IMAGE_EXAMPLE_ANSWER,\n Modality.AUDIO: AUDIO_SENTENCE_EXAMPLE_ANSWER if use_sentence_answers else AUDIO_EXAMPLE_ANSWER,\n }\n\n example_answer = \"\"\n for input_modality in input_modalities:\n example_answer += input_modality_to_example_answer[input_modality].rstrip()\n if use_sentence_answers:\n example_answer += \" \"\n example_answer = example_answer + \"\\n\"\n\n return example_answer\n\n def _get_all_format_kwargs(\n self,\n candidate: DataRecord | list[DataRecord],\n input_fields: list[str],\n input_modalities: set[Modality],\n output_fields: list[str],\n right_candidate: DataRecord | None,\n right_input_fields: list[str],\n right_input_modalities: set[Modality],\n **kwargs,\n ) -> dict:\n \"\"\"\n Returns a dictionary containing all the format kwargs for templating the prompts.\n\n Args:\n candidate (DataRecord): The input record.\n input_fields (list[str]): The input fields.\n output_fields (list[str]): The output fields.\n kwargs: The keyword arguments provided by the user.\n\n Returns:\n dict: The dictionary containing all the format kwargs.\n \"\"\"\n # get format kwargs which depend on the input data\n input_format_kwargs = {\n \"context\": self._get_context(candidate, input_fields),\n \"input_fields_desc\": self._get_input_fields_desc(candidate[0] if isinstance(candidate, list) else candidate, input_fields),\n \"output_fields_desc\": self._get_output_fields_desc(output_fields, **kwargs),\n \"agg_instruction\": self._get_agg_instruction(**kwargs),\n \"filter_condition\": self._get_filter_condition(**kwargs),\n \"join_condition\": self._get_join_condition(**kwargs),\n \"original_output\": self._get_original_output(**kwargs),\n \"critique_output\": self._get_critique_output(**kwargs),\n \"model_responses\": self._get_model_responses(**kwargs),\n \"chunk_outputs\": self._get_chunk_outputs(**kwargs),\n }\n\n # if a right candidate is provided, we also get the context and input field descriptions for the right candidate\n if right_candidate is not None:\n input_format_kwargs.update({\n \"right_context\": self._get_context(right_candidate, right_input_fields),\n \"right_input_fields_desc\": self._get_input_fields_desc(right_candidate, right_input_fields),\n })\n\n # get format kwargs which depend on the prompt strategy\n full_input_modalities = input_modalities.union(right_input_modalities)\n prompt_strategy_format_kwargs = {\n \"output_format_instruction\": self._get_output_format_instruction(),\n \"job_instruction\": self._get_job_instruction(full_input_modalities),\n \"desc_section\": self._get_desc_section(),\n \"critique_criteria\": self._get_critique_criteria(),\n \"refinement_criteria\": self._get_refinement_criteria(),\n \"finish_instruction\": self._get_finish_instruction(),\n \"example_input_fields\": self._get_example_input_fields(input_modalities),\n \"right_example_input_fields\": self._get_example_input_fields(right_input_modalities, right=True),\n \"example_output_fields\": self._get_example_output_fields(input_modalities),\n \"example_context\": self._get_example_context(input_modalities),\n \"second_example_context\": self._get_example_context(input_modalities, second=True) if self.prompt_strategy.is_agg_prompt() else \"\",\n \"third_example_context\": self._get_example_context(input_modalities, third=True) if self.prompt_strategy.is_agg_prompt() else \"\",\n \"right_example_context\": self._get_example_context(right_input_modalities, right=True),\n \"image_disclaimer\": self._get_image_disclaimer(input_modalities, agg=self.prompt_strategy.is_agg_prompt()),\n \"audio_disclaimer\": self._get_audio_disclaimer(input_modalities, agg=self.prompt_strategy.is_agg_prompt()),\n \"right_image_disclaimer\": self._get_image_disclaimer(right_input_modalities, right=True),\n \"right_audio_disclaimer\": self._get_audio_disclaimer(right_input_modalities, right=True),\n \"example_agg_instruction\": EXAMPLE_AGG_INSTRUCTION,\n \"example_filter_condition\": EXAMPLE_FILTER_CONDITION,\n \"example_join_condition\": EXAMPLE_JOIN_CONDITION,\n \"example_reasoning\": self._get_example_reasoning(input_modalities),\n \"example_answer\": self._get_example_answer(input_modalities),\n }\n\n # return all format kwargs\n return {**input_format_kwargs, **prompt_strategy_format_kwargs}\n\n def _create_audio_messages(self, candidate: DataRecord | list[DataRecord], input_fields: list[str]) -> list[dict]:\n \"\"\"\n Parses the candidate record(s) and returns the audio messages for the chat payload.\n\n Args:\n candidate (DataRecord | list[DataRecord]): The input record(s).\n input_fields (list[str]): The list of input fields.\n\n Returns:\n list[dict]: The audio messages for the chat payload.\n \"\"\"\n # normalize type to be list[DataRecord]\n if isinstance(candidate, DataRecord):\n candidate = [candidate]\n\n # create a message for each audio recording in an input field with an audio (or list of audio) type\n audio_content = []\n for field_name in input_fields:\n for dr in candidate:\n field_value = dr[field_name]\n field_type = dr.get_field_type(field_name)\n\n # audio filepath (or list of audio filepaths)\n if field_type.annotation in [AudioFilepath, AudioFilepath | None, AudioFilepath | Any] and field_value is not None:\n with open(field_value, \"rb\") as f:\n base64_audio_str = base64.b64encode(f.read()).decode(\"utf-8\")\n audio_content.append(\n {\"type\": \"input_audio\", \"input_audio\": {\"data\": base64_audio_str, \"format\": \"wav\"}}\n )\n\n elif field_type.annotation in [list[AudioFilepath], list[AudioFilepath] | None, list[AudioFilepath] | Any]:\n for audio_filepath in field_value:\n if audio_filepath is None:\n continue\n with open(audio_filepath, \"rb\") as f:\n base64_audio_str = base64.b64encode(f.read()).decode(\"utf-8\")\n audio_content.append(\n {\"type\": \"input_audio\", \"input_audio\": {\"data\": base64_audio_str, \"format\": \"wav\"}}\n )\n\n # pre-encoded images (or list of pre-encoded images)\n elif field_type.annotation in [AudioBase64, AudioBase64 | None, AudioBase64 | Any] and field_value is not None:\n audio_content.append(\n {\"type\": \"input_audio\", \"input_audio\": {\"data\": field_value, \"format\": \"wav\"}}\n )\n\n elif field_type.annotation in [list[AudioBase64], list[AudioBase64] | None, list[AudioBase64] | Any]:\n for base64_audio in field_value:\n if base64_audio is None:\n continue\n audio_content.append(\n {\"type\": \"input_audio\", \"input_audio\": {\"data\": base64_audio, \"format\": \"wav\"}}\n )\n\n return [{\"role\": \"user\", \"type\": \"input_audio\", \"content\": audio_content}] if len(audio_content) > 0 else []\n\n def _create_image_messages(self, candidate: DataRecord | list[DataRecord], input_fields: list[str]) -> list[dict]:\n \"\"\"\n Parses the candidate record(s) and returns the image messages for the chat payload.\n\n Args:\n candidate (DataRecord | list[DataRecord]): The input record(s).\n input_fields (list[str]): The list of input fields.\n\n Returns:\n list[dict]: The image messages for the chat payload.\n \"\"\"\n # normalize type to be list[DataRecord]\n if isinstance(candidate, DataRecord):\n candidate = [candidate]\n\n # create a message for each image in an input field with an image (or list of image) type\n image_content = []\n for field_name in input_fields:\n for dr in candidate:\n field_value = dr[field_name]\n field_type = dr.get_field_type(field_name)\n\n # image filepath (or list of image filepaths)\n if field_type.annotation in [ImageFilepath, ImageFilepath | None, ImageFilepath | Any] and field_value is not None:\n with open(field_value, \"rb\") as f:\n base64_image_str = base64.b64encode(f.read()).decode(\"utf-8\")\n media_type = _detect_image_media_type(filepath=field_value, base64_data=base64_image_str)\n image_content.append(\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{media_type};base64,{base64_image_str}\"}}\n )\n\n elif field_type.annotation in [list[ImageFilepath], list[ImageFilepath] | None, list[ImageFilepath] | Any]:\n for image_filepath in field_value:\n if image_filepath is None:\n continue\n with open(image_filepath, \"rb\") as f:\n base64_image_str = base64.b64encode(f.read()).decode(\"utf-8\")\n media_type = _detect_image_media_type(filepath=image_filepath, base64_data=base64_image_str)\n image_content.append(\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{media_type};base64,{base64_image_str}\"}}\n )\n\n # image url (or list of image urls)\n elif field_type.annotation in [ImageURL, ImageURL | None, ImageURL | Any] and field_value is not None:\n image_content.append({\"type\": \"image_url\", \"image_url\": {\"url\": field_value}})\n\n elif field_type.annotation in [list[ImageURL], list[ImageURL] | None, list[ImageURL] | Any]:\n for image_url in field_value:\n if image_url is None:\n continue\n image_content.append({\"type\": \"image_url\", \"image_url\": {\"url\": image_url}})\n\n # pre-encoded images (or list of pre-encoded images)\n elif field_type.annotation in [ImageBase64, ImageBase64 | None, ImageBase64 | Any] and field_value is not None:\n media_type = _detect_image_media_type(base64_data=field_value)\n image_content.append(\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{media_type};base64,{field_value}\"}}\n )\n\n elif field_type.annotation in [list[ImageBase64], list[ImageBase64] | None, list[ImageBase64] | Any]:\n for base64_image in field_value:\n if base64_image is None:\n continue\n media_type = _detect_image_media_type(base64_data=base64_image)\n image_content.append(\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:{media_type};base64,{base64_image}\"}}\n )\n\n return [{\"role\": \"user\", \"type\": \"image\", \"content\": image_content}] if len(image_content) > 0 else []\n\n def _get_system_prompt(self, **format_kwargs) -> str | None:\n \"\"\"\n Returns the fully templated system prompt for the given prompt strategy.\n Returns None if the prompt strategy does not use a system prompt.\n\n Returns:\n str | None: The fully templated system prompt (or None if the prompt strategy\n does not use a system prompt).\n \"\"\"\n base_prompt: str = self.BASE_SYSTEM_PROMPT_MAP.get(self.prompt_strategy)\n\n # for critic and refine prompt strategies, we do not use a base prompt\n if base_prompt is None:\n return base_prompt\n\n return base_prompt.format(**format_kwargs)\n\n def _get_user_messages(self, candidate: DataRecord | list[DataRecord], input_fields: list[str], right_candidate: DataRecord | None, right_input_fields: list[str], **kwargs) -> str:\n \"\"\"\n Returns a list of messages for the chat payload based on the prompt strategy.\n\n Args:\n candidate (DataRecord | list[DataRecord]): The input record(s).\n input_fields (list[str]): The input fields.\n output_fields (list[str]): The output fields.\n kwargs: The formatting kwargs and some keyword arguments provided by the user.\n\n Returns:\n Tuple[str, str | None]: The fully templated start and end of the user prompt.\n The second element will be None for text prompts.\n \"\"\"\n # get the base prompt template\n base_prompt = self.BASE_USER_PROMPT_MAP.get(self.prompt_strategy)\n\n # get any image messages for the chat payload (will be an empty list if no image fields exist)\n image_messages = self._create_image_messages(candidate, input_fields)\n\n # get any audio messages for the chat payload (will be an empty list if no audio fields exist)\n audio_messages = self._create_audio_messages(candidate, input_fields)\n\n # get any right image / audio messages for the chat payload (will be an empty list if image / audio not present)\n right_image_messages, right_audio_messages = [], []\n if self.prompt_strategy.is_join_prompt():\n assert right_candidate is not None, \"Right candidate must be provided for join prompts.\"\n right_image_messages = self._create_image_messages(right_candidate, right_input_fields)\n right_audio_messages = self._create_audio_messages(right_candidate, right_input_fields)\n\n # get any original messages for critique and refinement operations\n original_messages = kwargs.get(\"original_messages\")\n if self.prompt_strategy.is_critic_prompt() or self.prompt_strategy.is_refine_prompt():\n assert original_messages is not None, (\n \"Original messages must be provided for critique and refinement operations.\"\n )\n\n # combine image and audio messages\n image_audio_messages = image_messages + audio_messages\n right_image_audio_messages = right_image_messages + right_audio_messages\n has_image_audio = len(image_audio_messages) > 0\n has_right_image_audio = len(right_image_audio_messages) > 0\n\n # construct the user messages based on the prompt strategy\n user_messages = []\n if self.prompt_strategy.is_critic_prompt() or self.prompt_strategy.is_refine_prompt():\n # NOTE: if this critic / refinement prompt is processing images / audio, those images / audio\n # will be part of the `original_messages` and will show up in the final chat payload\n base_prompt_start, base_prompt_end = base_prompt.split(\"<>\\n\")\n user_messages.append({\"role\": \"user\", \"type\": \"text\", \"content\": base_prompt_start.format(**kwargs)})\n user_messages.extend(original_messages)\n user_messages.append({\"role\": \"user\", \"type\": \"text\", \"content\": base_prompt_end.format(**kwargs)})\n\n # handle joins with left and right images / audio\n elif self.prompt_strategy.is_join_prompt() and has_image_audio and has_right_image_audio:\n base_prompt_start, base_prompt_rest = base_prompt.split(\"<>\")\n base_prompt_mid, base_prompt_end = base_prompt_rest.split(\"<>\")\n user_messages.append({\"role\": \"user\", \"type\": \"text\", \"content\": base_prompt_start.format(**kwargs)})\n user_messages.extend(image_audio_messages)\n user_messages.append({\"role\": \"user\", \"type\": \"text\", \"content\": base_prompt_mid.format(**kwargs)})\n user_messages.extend(right_image_audio_messages)\n user_messages.append({\"role\": \"user\", \"type\": \"text\", \"content\": base_prompt_end.format(**kwargs)})\n\n # handle joins with only left images / audio\n elif self.prompt_strategy.is_join_prompt() and has_image_audio and not has_right_image_audio:\n base_prompt = base_prompt.replace(\"<>\", \"\")\n base_prompt_start, base_prompt_end = base_prompt.split(\"<>\")\n user_messages.append({\"role\": \"user\", \"type\": \"text\", \"content\": base_prompt_start.format(**kwargs)})\n user_messages.extend(image_audio_messages)\n user_messages.append({\"role\": \"user\", \"type\": \"text\", \"content\": base_prompt_end.format(**kwargs)})\n\n # handle joins with only right images / audio\n elif self.prompt_strategy.is_join_prompt() and not has_image_audio and has_right_image_audio:\n base_prompt = base_prompt.replace(\"<>\", \"\")\n base_prompt_start, base_prompt_end = base_prompt.split(\"<>\")\n user_messages.append({\"role\": \"user\", \"type\": \"text\", \"content\": base_prompt_start.format(**kwargs)})\n user_messages.extend(right_image_audio_messages)\n user_messages.append({\"role\": \"user\", \"type\": \"text\", \"content\": base_prompt_end.format(**kwargs)})\n\n # handle non-joins with images / audio\n elif not self.prompt_strategy.is_join_prompt() and has_image_audio and not self.prompt_strategy.is_moa_aggregator_prompt():\n base_prompt_start, base_prompt_end = base_prompt.split(\"<>\")\n user_messages.append({\"role\": \"user\", \"type\": \"text\", \"content\": base_prompt_start.format(**kwargs)})\n user_messages.extend(image_audio_messages)\n user_messages.append({\"role\": \"user\", \"type\": \"text\", \"content\": base_prompt_end.format(**kwargs)})\n\n # handle prompts w/no images or audio\n else:\n base_prompt = base_prompt.replace(\"<>\", \"\")\n base_prompt = base_prompt.replace(\"<>\", \"\")\n user_messages.append({\"role\": \"user\", \"type\": \"text\", \"content\": base_prompt.format(**kwargs)})\n\n return user_messages\n\n def create_messages(self, candidate: DataRecord | list[DataRecord], output_fields: list[str], right_candidate: DataRecord | None = None, **kwargs) -> list[dict]:\n \"\"\"\n Creates the messages for the chat payload based on the prompt strategy.\n\n Each message will be a dictionary with the following format:\n {\n \"role\": \"user\" | \"system\",\n \"type\": \"text\" | \"image\",\n \"content\": str\n }\n\n Args:\n candidate (DataRecord | list[DataRecord]): The input record(s).\n output_fields (list[str]): The output fields.\n right_candidate (DataRecord | None): The other join input record (only provided for joins).\n kwargs: The keyword arguments provided by the user.\n\n Returns:\n list[dict]: The messages for the chat payload.\n \"\"\"\n # compute the set of input fields\n input_fields = self._get_input_fields(candidate[0] if isinstance(candidate, list) else candidate, **kwargs)\n right_input_fields = [] if right_candidate is None else self._get_input_fields(right_candidate, **kwargs)\n\n # use input fields to determine the left / right input modalities\n input_modalities = self._get_input_modalities(candidate[0] if isinstance(candidate, list) else candidate, input_fields)\n right_input_modalities = set() if right_candidate is None else self._get_input_modalities(right_candidate, right_input_fields)\n\n # initialize messages\n messages = []\n\n # compute the full dictionary of format kwargs and add to kwargs\n format_kwargs = self._get_all_format_kwargs(candidate, input_fields, input_modalities, output_fields, right_candidate, right_input_fields, right_input_modalities, **kwargs)\n kwargs = {**kwargs, **format_kwargs}\n\n # generate system message (if applicable)\n system_prompt = self._get_system_prompt(**kwargs)\n if system_prompt is not None:\n messages.append({\"role\": \"system\", \"type\": \"text\", \"content\": system_prompt})\n\n # generate user messages and add to messages\n user_messages = self._get_user_messages(candidate, input_fields, right_candidate, right_input_fields, **kwargs)\n messages.extend(user_messages)\n\n return messages\n" }, { "path": "src/palimpzest/prompts/prompt_manager.py", "content": "\"\"\"\nPrompt caching utility for different LLM providers.\n\nThis module provides provider-specific prompt caching configurations:\n- OpenAI: Automatic prefix caching with prompt_cache_key for sticky routing\n- Gemini (Google AI Studio / Vertex AI): Implicit caching (automatic prefix matching)\n- Anthropic: Explicit cache_control with ephemeral type on system and user message content\n\"\"\"\n\nimport copy\nimport uuid\nfrom typing import Any\n\nfrom palimpzest.constants import Model\n\n\nclass PromptManager:\n \"\"\"\n Manages prompt caching configurations and message transformations for LLM providers.\n\n This class handles:\n 1. Session-level state (e.g., OpenAI cache keys).\n 2. Provider-specific request arguments (headers, extra_body).\n 3. Transformation of messages for providers requiring explicit markers (Anthropic).\n 4. Normalization of usage statistics.\n \"\"\"\n \n CACHE_BOUNDARY_MARKER = \"<>\"\n\n def __init__(self, model: Model):\n self.model = model\n # Instance-level state ensures thread safety if we use one manager per plan/execution\n self.openai_cache_key = f\"pz-cache-{uuid.uuid4().hex[:12]}\" if (self.model.is_provider_openai() or self.model.is_provider_azure()) else None\n\n def get_cache_kwargs(self) -> dict[str, Any]:\n \"\"\"\n Get provider-specific cache configuration kwargs for litellm.completion().\n\n Returns:\n A dictionary of kwargs to pass to litellm.completion() for enabling caching\n \"\"\"\n if not self.model.supports_prompt_caching():\n return {}\n # OpenAI and Azure OpenAI: https://platform.openai.com/docs/guides/prompt-caching\n # Use prompt_cache_key for sticky routing to the same cache shard\n if self.model.is_provider_openai() or self.model.is_provider_azure():\n return {\"extra_body\": {\"prompt_cache_key\": self.openai_cache_key}}\n else:\n return {}\n \n def inject_cache_isolation_id(self, messages: list[dict], session_id: str) -> list[dict]:\n \"\"\"\n Inject a cache isolation ID into messages for testing cache behavior per-modality.\n\n This must happen BEFORE update_messages_for_caching so the ID becomes part of cached content.\n \"\"\"\n for msg in messages:\n role = msg.get(\"role\")\n content = msg.get(\"content\")\n if role == \"system\" and isinstance(content, str) or \\\n role == \"user\" and self.model.is_provider_anthropic() and msg.get(\"type\") == \"text\" and isinstance(content, str):\n msg[\"content\"] = f\"[{session_id}] \" + content\n return messages\n\n def update_messages_for_caching(self, messages: list[dict]) -> list[dict]:\n \"\"\"\n Transform messages to conform to provider-specific caching requirements.\n\n - Anthropic: Adds explicit cache_control markers.\n - Others: Removes the generic cache boundary markers.\n\n Returns:\n The transformed messages list.\n \"\"\"\n if not self.model.supports_prompt_caching():\n return messages\n\n # Anthropic: Explicit cache_control with ephemeral type\n # https://platform.claude.com/docs/en/build-with-claude/prompt-caching\n if self.model.is_provider_anthropic():\n return self._transform_messages_for_anthropic(messages)\n # implicit caching for Gemini/OpenAI/Azure models that currently support caching\n # OpenAI: https://platform.openai.com/docs/guides/prompt-caching\n # Gemini: https://ai.google.dev/gemini-api/docs/caching\n elif (self.model.is_provider_openai() or self.model.is_provider_azure() or\n self.model.is_provider_google_ai_studio() or self.model.is_provider_vertex_ai()):\n return self._remove_cache_boundary_markers(messages)\n\n return messages\n\n\n def extract_usage_stats(self, usage: dict, is_audio_op: bool) -> dict[str, int]:\n \"\"\"\n Normalize cache statistics from provider-specific response formats.\n \"\"\"\n stats = {\n \"input_text_tokens\": 0,\n \"input_image_tokens\": 0, # forward looking\n \"input_audio_tokens\": 0,\n \"cache_creation_tokens\": 0,\n \"cache_read_tokens\": 0\n }\n\n details = usage.get(\"prompt_tokens_details\") or {}\n\n if self.model.is_provider_openai() or self.model.is_provider_azure():\n # only realtime audio models do, but they are not supported by PZ\n if self.model.supports_prompt_caching() and not is_audio_op:\n stats[\"cache_read_tokens\"] = details.get(\"cached_tokens\") or 0\n stats[\"input_text_tokens\"] = (usage.get(\"prompt_tokens\") or 0) - stats[\"cache_read_tokens\"]\n # audio models don't support caching for now\n elif is_audio_op:\n stats[\"input_text_tokens\"] = details.get(\"text_tokens\") or 0\n stats[\"input_audio_tokens\"] = details.get(\"audio_tokens\") or 0\n else:\n stats[\"input_text_tokens\"] = usage.get(\"prompt_tokens\") or 0\n\n # Moved to Gemini client class, now usage stats are extracted directly in GeminiClient\n # elif self.model.is_provider_vertex_ai():\n # stats[\"cache_read_tokens\"] = usage.get(\"cache_read_input_tokens\") or 0\n # if stats[\"cache_read_tokens\"] == 0:\n # stats[\"cache_read_tokens\"] = details.get(\"cached_tokens\") or 0\n # stats[\"input_text_tokens\"] = details.get(\"text_tokens\") or 0\n # stats[\"input_audio_tokens\"] = details.get(\"audio_tokens\") or 0\n # stats[\"input_image_tokens\"] = details.get(\"image_tokens\") or 0\n\n elif self.model.is_provider_anthropic():\n stats[\"cache_creation_tokens\"] = usage.get(\"cache_creation_input_tokens\") or 0\n stats[\"cache_read_tokens\"] = usage.get(\"cache_read_input_tokens\") or 0\n stats[\"input_text_tokens\"] = max(0, (usage.get(\"prompt_tokens\") or 0) - stats[\"cache_read_tokens\"] - stats[\"cache_creation_tokens\"])\n\n elif self.model.is_vllm_model():\n # vLLM does not seem to provide cache statistics through litellm, so we currently have no way\n # to extract cache read/creation tokens for vLLM models.\n pass\n\n # all other models (assume caching not supported)\n else:\n if is_audio_op:\n stats[\"input_text_tokens\"] = details.get(\"text_tokens\") or 0\n stats[\"input_audio_tokens\"] = details.get(\"audio_tokens\") or 0\n else:\n stats[\"input_text_tokens\"] = usage.get(\"prompt_tokens\") or 0\n\n\n return stats\n\n\n def _remove_cache_boundary_markers(self, messages: list[dict]) -> list[dict]:\n \"\"\"\n Remove <> markers from user messages.\n\n For providers with automatic (implicit) caching (OpenAI, Gemini), we don't need\n explicit cache markers. This function cleans up the markers from prompts.\n\n Args:\n messages: The list of messages to transform.\n\n Returns:\n A new list of messages with cache boundary markers removed.\n \"\"\"\n result = []\n for message in messages:\n new_message = message.copy()\n if new_message.get(\"role\") == \"user\":\n content = new_message.get(\"content\", \"\")\n if isinstance(content, str) and self.CACHE_BOUNDARY_MARKER in content:\n new_message[\"content\"] = content.replace(self.CACHE_BOUNDARY_MARKER, \"\")\n result.append(new_message)\n return result\n\n\n def _transform_messages_for_anthropic(self, messages: list[dict]) -> list[dict]:\n \"\"\"\n Add cache_control markers to system messages and user prompt prefixes for Anthropic models.\n\n This transforms messages to:\n 1. Add cache_control to system message content blocks\n 2. Convert user messages with <> marker into a single message with multiple content blocks:\n a. Static prefix block (with cache_control) - cacheable across records\n b. Dynamic content block (without cache_control) - changes per record\n\n Args:\n messages: The list of messages to transform.\n\n Returns:\n A new list of messages with cache_control markers added.\n \"\"\"\n result = []\n for message in messages:\n new_message = copy.deepcopy(message)\n role = new_message.get(\"role\")\n content = new_message.get(\"content\", \"\")\n\n # 1. Handle System Messages\n if role == \"system\":\n if isinstance(content, str) and content:\n new_message[\"content\"] = [{\n \"type\": \"text\",\n \"text\": content,\n \"cache_control\": {\"type\": \"ephemeral\"}\n }]\n elif isinstance(content, list) and content:\n # Apply to last block if it's text\n last_block = new_message[\"content\"][-1]\n if isinstance(last_block, dict) and last_block.get(\"type\") == \"text\":\n last_block[\"cache_control\"] = {\"type\": \"ephemeral\"}\n\n # 2. Handle User Messages (The Split Logic)\n elif role == \"user\" and isinstance(content, str) and self.CACHE_BOUNDARY_MARKER in content:\n static, dynamic = content.split(self.CACHE_BOUNDARY_MARKER, 1)\n\n new_blocks = []\n if static.strip():\n new_blocks.append({\n \"type\": \"text\",\n \"text\": static,\n \"cache_control\": {\"type\": \"ephemeral\"}\n })\n\n if dynamic.strip():\n new_blocks.append({\"type\": \"text\", \"text\": dynamic})\n\n if new_blocks:\n new_message[\"content\"] = new_blocks\n else:\n new_message[\"content\"] = \"\"\n\n result.append(new_message)\n return result\n" }, { "path": "src/palimpzest/prompts/split_merge_prompts.py", "content": "\"\"\"This file contains prompts for SplitConvert aggregator operations.\"\"\"\n\n### SYSTEM PROMPTS ###\nMAP_SPLIT_MERGER_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to generate a JSON object.\nYou will be presented with one or more outputs produced by a set of models operating on chunks of an input. Your task is to synthesize these responses into a single, high-quality JSON object which fills in the output fields with the correct values.\nIt is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased, incorrect, or contain duplicates.\n\nYou will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the model responses.\n\n{output_format_instruction} Finish your response with a newline character followed by ---\n\nAn example is shown below:\n---\nCHUNK 1 OUTPUT: the text mentions the scientists \"Augusta Ada King, Countess of Lovelace\" and \"Charles Babbage\". It states that King was an English mathematician who worked on Babbage's Analytical Engine.\n\nCHUNK 2 OUTPUT: the text passage mentions the scientist \"Charles Babbage\", who was a mathematician. Therefore, the name output should be [\"Charles Babbage\"] and the field_of_study output should be [\"Mathematician\"].\n\nINPUT FIELDS:\n- text: a text passage describing scientists\n\nOUTPUT FIELDS:\n- name: the list of names for each scientist mentioned in the text\n- field_of_study: a list with the field of study for each scientist\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: Looking at both chunk outputs, they specify that the scientists' formal names are \"Augusta Ada King\" and \"Charles Babbage\". Chunk Output 2 indicates that Charles Babbage was a Mathematician and Chunk Output 1 says that Augusta Ada King was an English mathematician. Therefore, the name output should be [\"Augusta Ada King\", \"Charles Babbage\"] and the field_of_study output should be [\"Mathematician\", \"Mathematician\"].\n\nANSWER:\n{{\n \"name\": [\"Augusta Ada King\", \"Charles Babbage\"],\n \"field_of_study\": [\"Mathematician\", \"Mathematician\"]\n}}\n---\n\"\"\"\n\nFILTER_SPLIT_MERGER_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to answer a TRUE/FALSE question.\nYou will be presented with one or more outputs produced by a set of models operating on chunks of an input. Your task is to synthesize these responses into a single TRUE/FALSE answer.\nIt is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased, incorrect, or contain duplicates.\n\nYou will be provided with a description of each input field and the filter condition.\n\nRemember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by ---\n\nAn example is shown below:\n---\nCHUNK 1 OUTPUT: The context describes Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace, who is widely recognized as a foundational figure in computer science. Therefore, the answer is TRUE.\n\nCHUNK 2 OUTPUT: Based on the context provided, Ada Lovelace is indeed a foundational computer scientist, therefore the answer is TRUE.\n\nINPUT FIELDS:\n- text: a text passage describing a scientist\n- birthday: the scientist's birthday\n- image: an image of the scientist\n- recording: an audio recording of a newscast about the scientist's contributions to their field\n\nFILTER CONDITION: The subject of the input is a foundational computer scientist.\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: Looking at both chunk outputs, they agree that the subject is a foundational computer scientist. Both outputs provide consistent evidence supporting this conclusion.\n\nANSWER: TRUE\n---\n\"\"\"\n\n### USER / INSTANCE-SPECIFIC PROMPTS ###\nMAP_SPLIT_MERGER_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to generate a JSON object.\nYou will be presented with one or more outputs produced by a set of models. Your task is to synthesize these responses into a single, high-quality JSON object which fills in the output fields with the correct values.\nIt is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased, incorrect, or contain duplicates.\n\nYou will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the model responses.\n\n{output_format_instruction} Finish your response with a newline character followed by ---\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nOUTPUT FIELDS:\n{output_fields_desc}\n\n<>{chunk_outputs}\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: \"\"\"\n\nFILTER_SPLIT_MERGER_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to answer a TRUE/FALSE question.\nYou will be presented with one or more outputs produced by a set of models operating on chunks of an input. Your task is to synthesize these responses into a single TRUE/FALSE answer.\nIt is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased, incorrect, or contain duplicates.\n\nYou will be provided with a description of each input field and the filter condition.\n\nRemember, your answer must be TRUE or FALSE. Finish your response with a newline character followed by ---\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nFILTER CONDITION: {filter_condition}\n\n<>{chunk_outputs}\n\nLet's think step-by-step in order to answer the question.\n\nREASONING: \"\"\"\n" }, { "path": "src/palimpzest/prompts/split_proposer_prompts.py", "content": "\"\"\"This file contains prompts for SplitAndMerge operations.\"\"\"\n\n### SYSTEM PROMPTS ###\nMAP_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a set of output fields to generate. Your task is to generate a detailed and succinct analysis describing what you believe is the correct value for each output field.\nBe sure to cite information from the context as evidence of why your answers are correct. Do not hallucinate evidence.\n\nYou will be provided with a description of each input field and each output field.\n\nAn example is shown below:\n---\nINPUT FIELDS:\n{example_input_fields}\n\nOUTPUT FIELDS:\n{example_output_fields}\n\nCONTEXT:\n{{{example_context}}}{image_disclaimer}{audio_disclaimer}\n\nLet's think step-by-step in order to answer the question.\n\nANSWER: {example_answer}\n---\n\"\"\"\n\nFILTER_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a filter condition. Your task is to generate a detailed and succinct analysis describing whether you believe the input satisfies the filter condition.\nBe sure to cite information from the context as evidence of why your determination is correct. Do not hallucinate evidence.\n\nYou will be provided with a description of each input field.\n\nAn example is shown below:\n---\nINPUT FIELDS:\n{example_input_fields}\n\nFILTER CONDITION: {example_filter_condition}\n\nCONTEXT:\n{{{example_context}}}{image_disclaimer}{audio_disclaimer}\n\nLet's think step-by-step in order to answer the question.\n\nANSWER: {example_answer}\n---\n\"\"\"\n\n### USER / INSTANCE-SPECIFIC PROMPTS ###\nMAP_SPLIT_PROPOSER_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a set of output fields to generate. Your task is to generate a paragraph or two which describes what you believe is the correct value for each output field.\nBe sure to cite information from the context as evidence of why your answers are correct. Do not hallucinate evidence.\n{desc_section}\nYou will be provided with a description of each input field and each output field.\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nOUTPUT FIELDS:\n{output_fields_desc}\n\n<>CONTEXT:\n{context}<>\n\nLet's think step-by-step in order to answer the question.\n\nANSWER: \"\"\"\n\nFILTER_SPLIT_PROPOSER_BASE_USER_PROMPT = \"\"\"You are a helpful assistant whose job is to {job_instruction}.\nYou will be presented with a context and a filter condition. Your task is to generate a detailed and succinct analysis describing whether you believe the input satisfies the filter condition.\nBe sure to cite information from the context as evidence of why your determination is correct. Do not hallucinate evidence.\n{desc_section}\nYou will be provided with a description of each input field.\n\nAn example is shown below:\n---\nINPUT FIELDS:\n{input_fields_desc}\n\nFILTER CONDITION: {filter_condition}\n\n<>CONTEXT:\n{context}<>\n\nLet's think step-by-step in order to answer the question.\n\nANSWER: \"\"\"\n" }, { "path": "src/palimpzest/prompts/utils.py", "content": "\"\"\"This file contains utility format strings which are templated into many of our prompts.\"\"\"\n\n### FORMATTING INSTRUCTIONS ###\nONE_TO_ONE_OUTPUT_FORMAT_INSTRUCTION = \"Remember, your answer must be a valid JSON dictionary. The dictionary should only have the specified output fields.\"\nONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION = \"Remember, your answer must be a valid JSON list of dictionaries. The list may contain one or more dictionaries, and each dictionary should only have the specified output fields.\"\n\n### USER-PROVIDED DESCRIPTION FOR MAPS / FILTERS / JOINS ###\nDESC_SECTION = \"\"\"\nThe user has additionally provided you with this description of the task you need to perform:\n{desc}\n\"\"\"\n\n### JOB INSTRUCTIONS ###\nAGG_JOB_INSTRUCTION = \"\"\"analyze input {modalities} in order to perform an aggregation and generate a JSON object\"\"\"\nMAP_JOB_INSTRUCTION = \"\"\"analyze input {modalities} in order to produce a JSON object\"\"\"\nFILTER_JOB_INSTRUCTION = \"\"\"analyze input {modalities} in order to answer a TRUE / FALSE question\"\"\"\nJOIN_JOB_INSTRUCTION = \"\"\"analyze input {modalities} in order to determine whether two data records satisfy a join condition\"\"\"\nPROPOSER_JOB_INSTRUCTION = \"\"\"analyze input {modalities} in order to produce an answer to a question\"\"\"\n\n### AGG / FILTER / JOIN CONDITIONS ###\nEXAMPLE_AGG_INSTRUCTION = \"Count the distinct number of scientists in the input.\"\nEXAMPLE_FILTER_CONDITION = \"The subject of the input is a foundational computer scientist.\"\nEXAMPLE_JOIN_CONDITION = \"The two inputs are scientists in the same academic field.\"\n\n### EXAMPLE INPUT FIELDS ###\nTEXT_EXAMPLE_INPUT_FIELDS = \"\"\"\n- text: a text passage describing a scientist\n- birthday: the scientist's birthday\n\"\"\"\nIMAGE_EXAMPLE_INPUT_FIELDS = \"\"\"\n- image: an image of the scientist\n- photographer: the photographer of the image\n\"\"\"\nAUDIO_EXAMPLE_INPUT_FIELDS = \"\"\"\n- recording: an audio recording of a newscast about the scientist's contributions to their field\n- speaker: the speaker in the recording\n\"\"\"\nRIGHT_TEXT_EXAMPLE_INPUT_FIELDS = \"\"\"\n- contents: the contents of a text file\n\"\"\"\nRIGHT_IMAGE_EXAMPLE_INPUT_FIELDS = \"\"\"\n- headshot: a headshot of a famous scientist\n\"\"\"\nRIGHT_AUDIO_EXAMPLE_INPUT_FIELDS = \"\"\"\n- podcast: an audio recording of a podcast about historic scientists\n\"\"\"\n\n### EXAMPLE OUTPUT FIELDS ###\nTEXT_EXAMPLE_OUTPUT_FIELDS = \"\"\"- name: the name of the scientist\n- birth_year: the year the scientist was born\"\"\"\nIMAGE_EXAMPLE_OUTPUT_FIELDS = \"\"\"- is_bald: true if the scientist is bald and false otherwise\"\"\"\nAUDIO_EXAMPLE_OUTPUT_FIELDS = \"\"\"- birthplace: the city where the scientist was born\"\"\"\nAGG_EXAMPLE_OUTPUT_FIELDS = \"\"\"- num_distinct_scientists: the number of distinct scientists mentioned in the input\"\"\"\n\n### EXAMPLE CONTEXTS ###\nTEXT_EXAMPLE_CONTEXT = \"\"\"\n \"text\": \"Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace, was an English mathematician and writer chiefly known for her work on Charles Babbage's proposed mechanical general-purpose computer, the Analytical Engine. She was the first to recognise that the machine had applications beyond pure calculation.\",\n \"birthday\": \"December 10, 1815\"\n\"\"\"\nIMAGE_EXAMPLE_CONTEXT = \"\"\"\n \"image\": ,\n \"photographer\": \"CameraEnthusiast1\"\n\"\"\"\nAUDIO_EXAMPLE_CONTEXT = \"\"\"\n \"recording\": ,\n \"speaker\": \"Walter Cronkite\"\n\"\"\"\nRIGHT_TEXT_EXAMPLE_CONTEXT = \"\"\"\n \"content\": \"Alan Turing was a pioneering computer scientist and mathematician. He is widely considered to be the father of theoretical computer science and artificial intelligence.\"\n\"\"\"\nRIGHT_IMAGE_EXAMPLE_CONTEXT = \"\"\"\n \"headshot\": \n\"\"\"\nRIGHT_AUDIO_EXAMPLE_CONTEXT = \"\"\"\n \"podcast\": \n\"\"\"\nSECOND_TEXT_EXAMPLE_CONTEXT = \"\"\"\n \"text\": \"Alan Turing was a pioneering computer scientist and mathematician. He is widely considered to be the father of theoretical computer science and artificial intelligence.\",\n \"birthday\": \"June 23, 1912\"\n\"\"\"\nSECOND_IMAGE_EXAMPLE_CONTEXT = \"\"\"\n \"image\": ,\n \"photographer\": \"PhotoPro42\"\n\"\"\"\nSECOND_AUDIO_EXAMPLE_CONTEXT = \"\"\"\n \"recording\": ,\n \"speaker\": \"Barbara Walters\"\n\"\"\"\nTHIRD_TEXT_EXAMPLE_CONTEXT = \"\"\"\n \"text\": \"Ada Lovelace is a historically significant computer scientist.\",\n \"birthday\": \"December 10, 1815\"\n\"\"\"\nTHIRD_IMAGE_EXAMPLE_CONTEXT = \"\"\"\n \"image\": ,\n \"photographer\": \"PicturePerfect\"\n\"\"\"\nTHIRD_AUDIO_EXAMPLE_CONTEXT = \"\"\"\n \"recording\": ,\n \"speaker\": \"Anderson Cooper\"\n\"\"\"\n\n### DISCLAIMERS ###\nIMAGE_DISCLAIMER = \"\"\"\n\\n\n\"\"\"\nAUDIO_DISCLAIMER = \"\"\"\n\\n