[ { "path": ".dockerignore", "content": ".tox\n.git\n.idea\n" }, { "path": ".github/workflows/ci.yml", "content": "---\nname: tron-ci\non:\n push:\n branches:\n - master\n tags:\n - v*.*\n pull_request:\n release:\njobs:\n tox:\n runs-on: ubuntu-22.04\n strategy:\n fail-fast: false\n matrix:\n toxenv:\n - py310,docs\n steps:\n - uses: actions/checkout@v2\n - uses: actions/setup-python@v2\n with:\n python-version: '3.10'\n # GHA won't setup tox for us\n - run: pip install tox==3.2\n # there are no pre-built wheels for bsddb3, so we need to install\n # its dpkg dependencies so that we can build a wheel when we're\n # creating our env. Once we get rid of bsddb3 as a Python dependency,\n # then we can also get rid of this dpkg\n - run: sudo apt-get install --quiet --assume-yes libdb5.3-dev\n # we explictly attempt to import the C extensions for some PyYAML\n # functionality, so we need the LibYAML bindings provided by this\n # package\n - run: sudo apt-get install --quiet --assume-yes libyaml-dev\n - run: tox -e ${{ matrix.toxenv }}\n build_debs:\n runs-on: ubuntu-22.04\n strategy:\n fail-fast: false\n matrix:\n dist:\n - jammy\n steps:\n - uses: actions/checkout@v2\n - uses: actions/setup-python@v2\n with:\n python-version: '3.10'\n # Update package lists to ensure we have the latest information\n - run: sudo apt-get update\n # the container provided by GitHub doesn't include utilities\n # needed for dpkg building, so we need to install `devscripts`\n # to bring those in\n - run: sudo apt-get install --quiet --assume-yes devscripts\n - run: make itest_${{ matrix.dist }}\n - uses: actions/upload-artifact@v4\n with:\n name: deb-${{ matrix.dist }}\n path: dist/tron_*.deb\n cut_release:\n runs-on: ubuntu-22.04\n needs: build_debs\n steps:\n - uses: actions/checkout@v2\n - run: mkdir -p dist/\n - uses: actions/download-artifact@v4\n with:\n name: deb-jammy\n path: dist/\n - name: Release\n uses: softprops/action-gh-release@v1\n if: startsWith(github.ref, 'refs/tags/v')\n with:\n generate_release_notes: true\n files: |\n dist/tron_*.deb\n fail_on_unmatched_files: true\n" }, { "path": ".github/workflows/security-review.yml", "content": "# Managed by terraform, do not edit manually\nname: Security Review\npermissions:\n pull-requests: write\n contents: read\n id-token: write\non:\n pull_request:\njobs:\n security:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n with:\n ref: ${{ github.event.pull_request.head.sha || github.sha }}\n fetch-depth: 2\n - name: Configure AWS credentials\n uses: aws-actions/configure-aws-credentials@v4\n with:\n role-to-assume: arn:aws:iam::${{ secrets.AWS_DEV_ACCOUNT_ID }}:role/security-review-bot\n aws-region: us-west-2\n - uses: anthropics/claude-code-security-review@0c6a49f1fa56a1d472575da86a94dbc1edb78eda\n with:\n comment-pr: true\n claude-api-key: \"github-actions\"\n claude-model: \"us.anthropic.claude-opus-4-6-v1\"\n run-every-commit: true\n env:\n CLAUDE_CODE_USE_BEDROCK: \"1\"\n AWS_REGION: \"us-west-2\"\n" }, { "path": ".gitignore", "content": "dist\nbuild\nMANIFEST\ntron.egg-info\n*.pyc\n*._*\n*.swp\n*.swo\ndocs/_build/\n.idea\n.vscode\n.fleet\ntron.iml\ndocs/images/\n*.dot\ntronweb/js/cs/*.js\nyarn.lock\ntronweb_tests/spec/*.js\ntronweb_tests/lib/\n.tox\n.tox-indocker\ntron.iml\n__pycache__/\n.pytest_cache/\ntron_state\ntron.lock\nmanhole.sock\nmanhole.sock.lock\nnode_modules/\n\n# Example cluster\nexample-cluster/config\nexample-cluster/MASTER.*\nexample-cluster/tron-repl.lock\nexample-cluster/tron_state*\nexample-cluster/manhole.sock*\nexample-cluster/_events/\n*.stdout\n*.stderr\ndev/manhole.sock.lock\ndev/tron.pid\ndev/_events/\n\n# Generated debian artifacts\ndebian/.debhelper/\ndebian/debhelper-build-stamp\ndebian/files\ndebian/tron\ndebian/tron.debhelper.log\ndebian/tron.postinst.debhelper\ndebian/tron.postrm.debhelper\ndebian/tron.preinst.debhelper\ndebian/tron.prerm.debhelper\ndebian/tron.substvars\n" }, { "path": ".pre-commit-config.yaml", "content": "---\ndefault_language_version:\n python: python3.10\nrepos:\n - repo: https://github.com/pre-commit/pre-commit-hooks\n rev: v2.5.0\n hooks:\n - id: trailing-whitespace\n - id: end-of-file-fixer\n exclude: CHANGELOG.md\n - id: check-docstring-first\n - id: check-json\n - id: check-yaml\n - id: requirements-txt-fixer\n - id: fix-encoding-pragma\n args: [--remove]\n - id: pretty-format-json\n args: [--autofix, --indent, '4', --no-sort-keys]\n - repo: https://github.com/PyCQA/flake8\n rev: 5.0.4\n hooks:\n - id: flake8\n exclude: ^docs/source/conf.py$\n - repo: https://github.com/asottile/reorder_python_imports\n rev: v1.9.0\n hooks:\n - id: reorder-python-imports\n args: [--py3-plus]\n - repo: https://github.com/asottile/pyupgrade\n rev: v3.20.0\n hooks:\n - id: pyupgrade\n args: [--py39-plus]\n - repo: local\n hooks:\n - id: patch-enforce-autospec\n name: mock.patch enforce autospec\n description: |\n This hook ensures all mock.patch invocations specify an autospec\n entry: contrib/mock_patch_checker.py\n language: script\n files: ^tests/.*\\.py$\n - repo: http://github.com/psf/black\n rev: 22.3.0\n hooks:\n - id: black\n args: [--target-version, py310]\n" }, { "path": ".pyautotest", "content": "test_runner_name: \"testify\"\n" }, { "path": ".readthedocs.yaml", "content": "# Read the Docs configuration file for Sphinx projects\n# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details\n\n# Required\nversion: 2\n\n# RTD defaults as of 2023-11-08\nbuild:\n os: ubuntu-22.04\n tools:\n python: \"3.10\"\n # You can also specify other tool versions:\n # nodejs: \"20\"\n # rust: \"1.70\"\n # golang: \"1.20\"\n\n# Also provide downloadable zip\nformats: [htmlzip]\n\n# Build documentation in the \"docs/\" directory with Sphinx\nsphinx:\n configuration: docs/source/conf.py\n # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs\n # builder: \"dirhtml\"\n # Fail on all warnings to avoid broken references\n # fail_on_warning: true\n\n# Optionally build your docs in additional formats such as PDF and ePub\n# formats:\n# - pdf\n# - epub\n\n# Optional but recommended, declare the Python requirements required\n# to build your documentation\n# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html\npython:\n install:\n - requirements: requirements-docs.txt\n" }, { "path": "AGENTS.md", "content": "# AGENTS.md\n\nThis file provides guidance to AI coding agents working with code in this repository.\n\n## What is Tron?\n\nTron is Yelp's centralized batch job scheduling system—a distributed alternative to cron for managing periodic batch processes across a cluster.\n\n### Core Concepts\n\n**Jobs** are DAGs (directed acyclic graphs) defined in YAML configuration. Each job contains one or more **Actions** (individual commands) with dependency relationships between them.\n\n**JobRuns** are instances of a job execution. If a job runs daily, each execution creates a new JobRun containing **ActionRuns** for each action. Tron tracks each run independently.\n\n### Lifecycle\n\n```mermaid\nflowchart LR\n A[Startup] --> B[Restore state from DynamoDB]\n B --> C[Schedule jobs]\n C --> D[Track state changes]\n D --> E[Save state to DynamoDB]\n E --> C\n```\n\n### State Persistence\n\nTron persists all job and run state to DynamoDB. 101 consecutive save errors triggers intentional crash (prevents running degraded).\n\n```mermaid\nflowchart LR\n A[State change] --> B[Buffer]\n B -->|buffer full| C[Save queue]\n C --> D[Pop from queue]\n D --> E[Delete existing entries]\n E --> F[Partition into chunks]\n F --> G[Batch write to DynamoDB]\n G -->|failure| C\n```\n\n### Execution Backends\n\n- **Kubernetes**: Primary execution backend\n- **SSH**: Legacy backend for remote command execution, do not extend\n- **Mesos**: Deprecated, do not extend\n\n## Project Structure\n\n```\ntron/\n├── core/ # Job, Action, JobRun, scheduling, dependency graphs\n├── config/ # YAML config parsing and schema definitions\n├── serialize/ # State persistence (DynamoDB, shelve backends)\n├── api/ # REST API endpoints and adapters\n├── kubernetes.py # Kubernetes execution backend\n└── mesos.py # Deprecated - do not modify\n\ntronweb/ # Web UI (CoffeeScript/Backbone.js)\nbin/ # CLI: trond, tronctl, tronview, tronfig\n```\n\n## Testing\n\nTox manages the virtualenv in `.tox/py310/`. Use `make test` for the full suite, or iterate with pytest directly:\n\n```bash\n.tox/py310/bin/pytest tests/path/to/test.py -x\n```\n\n## Development Guardrails\n\n### High-risk areas\n\n**DynamoDB/Persistence changes:**\n- Pickle deserialization is still active—deleting or renaming persisted classes/fields breaks restore\n- Reverting changes that add new persisted fields is NOT safe\n- Writes batch 8 partitions at a time; large jobs needing more can be partially written if a later batch fails\n\n**Job/action schema changes** require updates in two repos:\n- Tron: `tron/config/schema.py`, `tron/core/action.py` (including `from_json`/`to_json`)\n- PaaSTA: `paasta_tools/cli/schemas/tron_schema.json`, `paasta_tools/tron_tools.py`\n- Plus: tests in both repos, and any code that consumes the new field\n\n**MASTER config changes**:\n- `tron/config/schema.py` — Add field to config object\n- `tron/config/config_parse.py` — Add default value and validator\n- Plus: tests, and any code that consumes the new config value\n\nReverting config changes is risky: new params get written to MASTER.yaml on disk, so reverting code requires manual config cleanup on servers.\n\n### Do not modify\n\n- `tron/mesos.py` — Deprecated\n- `tron/ssh.py` - Deprecated\n- `tron/node.py` - Deprecated\n\n\n- DynamoDB schema without approval\n" }, { "path": "CODEOWNERS", "content": "# NOTE: \"we\" in this file will refer to the Compute Infrastructure team at Yelp\n* @Yelp/paasta\n#\n# prevent cheeky modifications :)\nCODEOWNERS @Yelp/paasta\n" }, { "path": "LICENSE.txt", "content": "Copyright 2010-2012 Yelp\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "MANIFEST.in", "content": "include *.txt\ninclude *.md\ninclude Makefile\ninclude tron/default_config.yaml\ninclude tron/logging.conf\ninclude tron/named_config_template.yaml\nrecursive-include tests *.py *.yaml\nrecursive-include docs *.rst *.yaml *.1 *.8\nrecursive-include tronweb *\nrecursive-exclude tronweb *.coffee\n" }, { "path": "Makefile", "content": "# Edit this release and run \"make release\"\nRELEASE=3.10.0\n\nSHELL=/bin/bash\n\nDOCKER_RUN = docker run -t -v $(CURDIR):/work:rw -v $(CURDIR)/.tox-indocker:/work/.tox:rw\nUID:=$(shell id -u)\nGID:=$(shell id -g)\n\nifeq ($(findstring .yelpcorp.com,$(shell hostname -f)), .yelpcorp.com)\n\tPAASTA_ENV ?= YELP\nelse\n\tPAASTA_ENV ?= $(shell hostname --fqdn)\nendif\n\nNOOP = true\nifeq ($(PAASTA_ENV),YELP)\n\texport PIP_INDEX_URL ?= http://169.254.255.254:20641/$*/simple/\n\tADD_MISSING_DEPS_MAYBE:=-diff --unchanged-line-format= --old-line-format= --new-line-format='%L' ./requirements.txt ./yelp_package/extra_requirements_yelp.txt >> ./requirements.txt\nelse\n\texport PIP_INDEX_URL ?= https://pypi.python.org/simple\n\tADD_MISSING_DEPS_MAYBE:=$(NOOP)\nendif\n\n.PHONY : all clean tests docs dev\n\n-usage:\n\t@echo \"make test - Run tests\"\n\t@echo \"make deb_jammy - Generate jammy deb package\"\n\t@echo \"make itest_jammy - Run tests and integration checks\"\n\t@echo \"make _itest_jammy - Run only integration checks\"\n\t@echo \"make release - Prepare debian info for new release\"\n\t@echo \"make clean - Get rid of scratch and byte files\"\n\t@echo \"make dev - Get a local copy of trond running in debug mode in the foreground\"\n\ndocker_%:\n\t@echo \"Building docker image for $*\"\n\t[ -d dist ] || mkdir -p dist\n\tcd ./yelp_package/$* && docker build --build-arg PIP_INDEX_URL=${PIP_INDEX_URL} -t tron-builder-$* .\n\ndeb_%: clean docker_% coffee_%\n\t@echo \"Building deb for $*\"\n\t# backup these files so we can temp modify them\n\tcp requirements.txt requirements.txt.old\n\t$(ADD_MISSING_DEPS_MAYBE)\n\t$(DOCKER_RUN) -e PIP_INDEX_URL=${PIP_INDEX_URL} tron-builder-$* /bin/bash -c ' \\\n\t\tdpkg-buildpackage -d && \\\n\t\tmv ../*.deb dist/ && \\\n\t\trm -rf debian/tron \\\n\t'\n\t# restore the backed up files\n\tmv requirements.txt.old requirements.txt\n\ncoffee_%: docker_%\n\t@echo \"Building tronweb\"\n\t$(DOCKER_RUN) tron-builder-$* /bin/bash -c ' \\\n\t\trm -rf tronweb/js/cs && \\\n\t\tmkdir -p tronweb/js/cs && \\\n\t\tcoffee -o tronweb/js/cs/ -c tronweb/coffee/ \\\n\t'\n\ntest:\n\ttox -e py310\n\ntest_in_docker_%: docker_%\n\t$(DOCKER_RUN) tron-builder-$* python3.10 -m tox -vv -e py310\n\ntox_%:\n\ttox -e $*\n\n_itest_%:\n\t$(DOCKER_RUN) ubuntu:$* /work/itest.sh\n\ndebitest_%: deb_% _itest_%\n\t@echo \"Package for $* looks good\"\n\nitest_%: debitest_%\n\t@echo \"itest $* OK\"\n\ndev:\n\tSSH_AUTH_SOCK=$(SSH_AUTH_SOCK) .tox/py310/bin/trond --debug --working-dir=dev -l logging.conf --host=0.0.0.0\n\nexample_cluster:\n\ttox -e example-cluster\n\nyelpy:\n\t.tox/py310/bin/pip install -r yelp_package/extra_requirements_yelp.txt\n\n\n# 1. Bump version at the top of this file\n# 2. `make release`\n\nVERSION = $(firstword $(subst -, ,$(RELEASE) ))\nLAST_COMMIT_MSG = $(shell git log -1 --pretty=%B | sed -e 's/\\x27/\"/g')\nrelease:\n\t@if [[ \"$$(git status --porcelain --untracked-files=no :^/Makefile)\" != '' ]]; then echo \"Error: Working directory is not clean; only changes to Makefile are allowed when cutting a release.\"; exit 1; fi\n\t$(eval untracked_files_tmpfile=$(shell mktemp))\n\tgit status --porcelain --untracked-files=all :^./Makefile > $(untracked_files_tmpfile)\n\t@if [[ \"$$(git status --porcelain --untracked-files=normal :/docs/source/generated)\" != '' ]]; then echo \"Error: Untracked files found in docs/source/generated.\"; exit 1; fi\n\t@if existing_sha=$$(git rev-parse --verify --quiet v$(VERSION)); then echo \"Error: tag v$(VERSION) exists and points at $$existing_sha\"; exit 1; fi\n\t@read upstream_master junk <<<\"$$(git ls-remote -h origin master)\" && if ! git merge-base --is-ancestor $$upstream_master HEAD; then echo \"Error: HEAD is missing commits from origin/master ($$upstream_master).\"; exit 1; fi\n\tdch -v $(RELEASE) --distribution jammy --changelog ./debian/changelog $$'$(VERSION) tagged with \\'make release\\'\\rCommit: $(LAST_COMMIT_MSG)'\n\tsed -i -e \"s/__version__ = .*/__version__ = \\\"$(VERSION)\\\"/\" ./tron/__init__.py\n\tmake docs || true\n\tgit add ./Makefile ./debian/changelog ./tron/__init__.py ./docs/source/generated/\n\tgit commit -m \"Released $(RELEASE) via make release\"\n\tif [[ \"$$(git status --porcelain --untracked-files=all)\" != \"$$(<$(untracked_files_tmpfile))\" ]]; then echo \"Error: automatic git commit left some files uncommitted. Fix the git commit command in ./Makefile to include any automatically generated files that it is currently missing.\"; exit 1; fi\n\tgit tag v$(VERSION)\n\tgit push --atomic origin master v$(VERSION)\n\ndocs:\n\ttox -r -e docs\n\nman:\n\twhich $(SPHINXBUILD) >/dev/null && $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(DOCS_DIR) $(DOCS_DIR)/source/man || true\n\t@echo\n\t@echo \"Build finished. The manual pages are in $(DOCS_BUILDDIR)/source/man.\"\n\nclean:\n\trm -rf tronweb/js/cs\n\tfind . -name '*.pyc' -delete\n" }, { "path": "OWNERS", "content": "---\nteams:\n- Compute Infra \n" }, { "path": "README.md", "content": "Tron - Batch Scheduling System\n==============================\n\n[![Build Status](https://github.com/Yelp/Tron/actions/workflows/ci.yml/badge.svg?query=branch%3Amaster)](https://github.com/Yelp/Tron/actions/workflows/ci.yml)\n[![Documentation Status](https://readthedocs.org/projects/tron/badge/?version=latest)](http://tron.readthedocs.io/en/latest/?badge=latest)\n\nTron is a centralized system for managing periodic batch processes\nacross a cluster. If you find [cron](http://en.wikipedia.org/wiki/Cron) or\n[fcron](http://fcron.free.fr/) to be insufficient for managing complex work\nflows across multiple computers, Tron might be for you.\n\nInstall with:\n\n > sudo pip install tron\n\nOr look at the [tutorial](http://tron.readthedocs.io/en/latest/tutorial.html).\n\nThe full documentation is available [on ReadTheDocs](http://tron.readthedocs.io/en/latest/).\n\nVersions / Roadmap\n------------------\n\nTron is changing and under active development.\n\nIt is being transformed from an ssh-based execution engine to be compatible with running on [Kubernetes\n](https://kubernetes.io/docs/concepts/overview/).\n\nTron development is specifically targeting Yelp's needs and not designed to be\na general solution for other companies.\n\n\nContributing\n------------\n\nRead [Working on Tron](http://tron.readthedocs.io/en/latest/developing.html) and\nstart sending pull requests!\n\nAny issues should be posted [on Github](http://github.com/Yelp/Tron/issues).\n\nBerkeleyDB on Mac OS X\n----------------------\n\n $ brew install berkeley-db\n $ export BERKELEYDB_DIR=$(brew --cellar)/berkeley-db/\n $ export YES_I_HAVE_THE_RIGHT_TO_USE_THIS_BERKELEY_DB_VERSION=1\n" }, { "path": "bin/generate_tron_tab_completion_cache", "content": "#!/usr/bin/env python\n\"\"\"\nprint a list of all the tron jobs, to be saved as a cache for tab completion\n\"\"\"\nimport argcomplete\n\nfrom tron.commands import cmd_utils\nfrom tron.commands.client import Client\n\n\ndef main():\n parser = cmd_utils.build_option_parser()\n argcomplete.autocomplete(parser)\n args = parser.parse_args()\n cmd_utils.load_config(args)\n\n client = Client(args.server)\n for job in client.jobs(include_job_runs=True, include_action_runs=True):\n print(job[\"name\"])\n for run in job[\"runs\"]:\n print(run[\"id\"])\n for action in run[\"runs\"]:\n print(action[\"id\"])\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "bin/tronctl", "content": "#!/usr/bin/env python\n\"\"\"Tron Control\n\nPart of the command line interface to the tron daemon. Provides the interface\nto controlling jobs and runs.\n\"\"\"\nimport argparse\nimport asyncio\nimport datetime\nimport logging\nimport pprint\nimport sys\nfrom collections import defaultdict\nfrom collections.abc import Callable\nfrom collections.abc import Generator\nfrom typing import Any\nfrom urllib.parse import urljoin\n\nimport argcomplete # type: ignore\n\nfrom tron import __version__\nfrom tron.commands import client\nfrom tron.commands import cmd_utils\nfrom tron.commands.backfill import BackfillRun\nfrom tron.commands.backfill import confirm_backfill\nfrom tron.commands.backfill import DEFAULT_MAX_PARALLEL_RUNS\nfrom tron.commands.backfill import get_date_range\nfrom tron.commands.backfill import LIMIT_MAX_PARALLEL_RUNS\nfrom tron.commands.backfill import print_backfill_cmds\nfrom tron.commands.backfill import print_backfill_runs_table\nfrom tron.commands.backfill import run_backfill_for_date_range\nfrom tron.commands.client import RequestError\nfrom tron.commands.client import TronObjectIdentifier\nfrom tron.commands.cmd_utils import COLOR_YELLOW\nfrom tron.commands.cmd_utils import ExitCode\nfrom tron.commands.cmd_utils import suggest_possibilities\nfrom tron.commands.cmd_utils import tron_jobs_completer\nfrom tron.commands.cmd_utils import warning_output\nfrom tron.commands.retry import parse_deps_timeout\nfrom tron.commands.retry import print_retries_table\nfrom tron.commands.retry import retry_actions\nfrom tron.commands.retry import RetryAction\n\nCOMMAND_HELP = (\n (\n \"start\",\n \"job name, job run id, or action id\",\n \"Start the selected job, job run, or action. Creates a new job run if starting a job.\",\n ),\n (\n \"rerun\",\n \"job run id\",\n \"Start a new job run with the same start time command context as the given job run.\",\n ),\n (\n \"retry\",\n \"action id\",\n \"Re-run a job action within an existing job run. Uses latest code/config except the command by default. Add --use-latest-command to use the latest command.\",\n ),\n (\"recover\", \"action id\", \"Ask Tron to start tracking an UNKNOWN action run again\"),\n (\"cancel\", \"job run id\", \"Cancel the selected job run.\"),\n (\n \"backfill\",\n \"job name\",\n \"Start job runs for a particular date range\",\n ),\n (\n \"disable\",\n \"job name\",\n \"Disable selected job and cancel any outstanding runs. WARNING: you *must* disable the job in yelpsoa-configs to guarantee it will not be re-enabled.\",\n ),\n (\"enable\", \"job name\", \"Enable the selected job and schedule the next run\"),\n (\n \"fail\",\n \"job run or action id\",\n \"Mark an UNKNOWN job or action as failed. Does not publish action triggers.\",\n ),\n (\n \"success\",\n \"job run or action id\",\n \"Mark an UNKNOWN job or action as having succeeded. Will publish action triggers.\",\n ),\n (\n \"skip\",\n \"action id\",\n \"Skip a failed action, unblocks dependent actions. Does *not* publish action triggers.\",\n ),\n (\n \"skip-and-publish\",\n \"action id\",\n \"Skip a failed action, unblocks dependent actions. *Does* publish action triggers.\",\n ),\n (\"stop\", \"action id\", \"Stop the action run (SIGTERM)\"),\n (\"kill\", \"action id\", \"Force kill the action run (SIGKILL)\"),\n (\"move\", \"job name\", \"Rename a job\"),\n (\"publish\", \"trigger id\", \"Publish actionrun trigger to kick off downstream jobs\"),\n (\"discard\", \"trigger id\", \"Discard existing actionrun trigger\"),\n (\"version\", None, \"Print tron client and server versions\"),\n)\n\nlog = logging.getLogger(\"tronctl\")\n\n\ndef parse_date(date_string):\n return datetime.datetime.strptime(date_string, \"%Y-%m-%d\")\n\n\ndef parse_cli():\n parser = cmd_utils.build_option_parser()\n subparsers = parser.add_subparsers(dest=\"command\", title=\"commands\", help=\"Tronctl command to run\", required=True)\n\n cmd_parsers = {}\n for cmd_name, id_help_text, desc in COMMAND_HELP:\n cmd_parsers[cmd_name] = subparsers.add_parser(cmd_name, help=desc, description=desc)\n if id_help_text:\n cmd_parsers[cmd_name].add_argument(\n \"id\", nargs=\"*\", help=id_help_text\n ).completer = cmd_utils.tron_jobs_completer\n\n # HACK: this is slightly funky since we already add --verbose in cmd_utils.build_option_parser(),\n # but that requires something like tronctl --verbose start JOB rather than tronctl start -v JOB\n cmd_parsers[cmd_name].add_argument(\n \"-v\",\n \"--verbose\",\n action=\"count\",\n help=\"Verbose logging\",\n default=None,\n )\n\n # start\n cmd_parsers[\"start\"].add_argument(\n \"--run-date\",\n type=parse_date,\n dest=\"run_date\",\n help=\"What the run-date should be set to\",\n )\n\n # backfill\n backfill_parser = cmd_parsers[\"backfill\"]\n mutex_dates_group = backfill_parser.add_mutually_exclusive_group(required=True)\n mutex_dates_group.add_argument(\n \"--start-date\",\n type=parse_date,\n dest=\"start_date\",\n help=\"First run-date to backfill\",\n )\n backfill_parser.add_argument(\n \"--end-date\",\n type=parse_date,\n dest=\"end_date\",\n help=(\n \"Last run-date to backfill (note: many jobs operate on date-1), \"\n \"assuming --start-date is set. This date is inclusive. Defaults to today.\"\n ),\n )\n backfill_parser.add_argument(\n \"--descending\",\n action=\"store_true\",\n default=False,\n help=(\n \"If set, backfill from end date to start date. Otherwise, \"\n \"the default is to backfill from start date to end date.\"\n ),\n )\n mutex_dates_group.add_argument(\n \"-d\",\n \"--dates\",\n type=lambda v: [parse_date(date_str.strip()) for date_str in v.split(\",\")],\n dest=\"dates\",\n help=(\n \"List of comma-separated dates to run backfills on. \"\n \"Backfills will be executed for dates in the order they are presented.\"\n ),\n )\n backfill_parser.add_argument(\n \"-P\",\n \"--max-parallel\",\n type=int,\n dest=\"max_parallel\",\n default=DEFAULT_MAX_PARALLEL_RUNS,\n help=(\n \"The max number of dates that can be backfilled in parallel. \"\n \"Before setting, consider how much in resources your job needs. \"\n \"If it needs a lot, keep this number low, because there may not be \"\n \"enough resources in the cluster too satisfy the demand, which can \"\n \"adversely affect other jobs. \"\n \"The default is %(default)s.\"\n ),\n )\n backfill_parser.add_argument(\n \"--fail-on-error\",\n dest=\"fail_on_error\",\n action=\"store_true\",\n default=False,\n help=(\n \"If set, the overall backfill will fail immediately if a backfill \"\n \"for a single date fails. All in-progress backfills will cancelled. \"\n \"If a single backfill is still considered successful it was otherwise \"\n \"cancelled or skipped by the user. \"\n \"By default, individual backfill failures are ignored.\"\n ),\n )\n backfill_parser.add_argument(\n \"--dry-run\",\n action=\"store_true\",\n default=False,\n help=\"Prints the equivalent `tronctl start` commands for the backfill\",\n )\n\n # retry\n retry_parser = cmd_parsers[\"retry\"]\n retry_parser.add_argument(\n \"--use-latest-command\",\n action=\"store_true\",\n default=False,\n help=\"Use the latest command in tronfig rather than the original command when the action run was created\",\n )\n retry_parser.add_argument(\n \"--wait-for-deps\",\n type=parse_deps_timeout,\n default=0,\n dest=\"deps_timeout\",\n help=(\n \"Max duration to wait for upstream dependencies (upstream triggers \"\n \"and/or same job actions) before attempting to retry. \"\n \"If all dependencies are not done when the timeout expires, \"\n \"this command will exit with an error, and the action will NOT be retried. \"\n \"Must be either an int number of seconds, a human-readable/\"\n \"pytimeparse-parsable string, or 'infinity' to wait forever. \"\n \"Defaults to 0 (don't wait).\"\n ),\n )\n\n argcomplete.autocomplete(parser)\n args = parser.parse_args()\n\n return args\n\n\ndef request(url: str, data: dict[str, Any], headers=None, method=None) -> bool:\n # We want every tronctl request to be attributable\n response = client.request(url, data=data, headers=headers, method=method, user_attribution=True)\n if response.error:\n print(f\"Error: {response.content}\")\n return False\n print(response.content.get(\"result\", \"OK\"))\n return True\n\n\ndef event_publish(args):\n for event in args.id:\n # trying to publish a job run/action run id will likely print multiple warnings\n # since the conditions are somewhat overlapping - only print the first one\n warning_printed = False\n split_event = event.split(\".\")\n # first, let's try to catch folks trying to publish a job run or action run id as a trigger\n # these will look something like NAMESPACE.JOB_NAME.RUN_NUMBER or NAMESPACE.JOB_NAME.RUN_NUMBER.ACTION_NAME\n # i.e., if the 3rd element is an integer, it's one of these\n if len(split_event) >= 3:\n try:\n int(split_event[2])\n print(\n warning_output(\n f\"\\nWarning: the event id '{event}' looks like a job run or action run id rather than a trigger id!\",\n color=COLOR_YELLOW,\n )\n )\n print(\n warning_output(\n \"This is almost certainly incorrect and you want something like `tronctl publish $SERVICE.$JOB_NAME.$ACTION_NAME.$TRIGGER_NAME.$TRIGGER_VALUE`\",\n color=COLOR_YELLOW,\n )\n )\n warning_printed = True\n except ValueError:\n pass\n\n if len(split_event) != 5 and not warning_printed:\n print(\n warning_output(\n f\"\\nWarning: '{event}' is too {'long' if len(split_event) > 5 else 'short'} and does not match the expected trigger format!\",\n color=COLOR_YELLOW,\n )\n )\n print(\n warning_output(\n \"This is almost certainly incorrect and you want something like `tronctl publish $SERVICE.$JOB_NAME.$ACTION_NAME.$TRIGGER_NAME.$TRIGGER_VALUE`\",\n color=COLOR_YELLOW,\n )\n )\n\n yield request(\n urljoin(args.server, \"/api/events\"),\n dict(command=\"publish\", event=event),\n )\n\n\ndef event_discard(args):\n for event in args.id:\n yield request(\n urljoin(args.server, \"/api/events\"),\n dict(command=\"discard\", event=event),\n )\n\n\ndef _get_triggers_for_action(server: str, action_identifier: str) -> tuple[str, ...] | None:\n try:\n namespace, job_name, run_number, action_name = action_identifier.split(\".\")\n except ValueError:\n print(\n f\"Unable to fully decompose {action_identifier}: expected an identifier of the form (namespace).(job).(run).(action)\"\n )\n return None\n\n trigger_response = client.request(\n uri=urljoin(\n server,\n f\"/api/jobs/{namespace}.{job_name}/{run_number}/{action_name}\",\n ),\n )\n if trigger_response.error:\n print(f\"Unable to fetch downstream triggers for {action_identifier}: {trigger_response.error}\")\n return None\n\n # triggers are returned by the API as comma-separated values with a space after every comma, which is\n # not automation-friendly - thus the non-standard multi-character split\n triggers = trigger_response.content.get(\"trigger_downstreams\", \"\").split(\", \")\n\n # the API will return an empty string for actions with no triggers to emit, but splitting '' yields [''],\n # so we want to make sure that we return an empty iterable in this case\n return tuple(f\"{namespace}.{job_name}.{action_name}.{trigger}\" for trigger in triggers if trigger)\n\n\ndef skip_and_publish(server: str, tron_id: TronObjectIdentifier, identifier: str) -> bool:\n all_success = True\n print(f\"Skipping {identifier}...\")\n if request(\n url=urljoin(server, tron_id.url),\n data={\"command\": \"skip\"},\n ):\n print(f\"Successfully skipped {identifier}.\")\n print(f\"\\nFetching triggers to publish for {identifier}...\")\n # a single action can have 0..N triggers to publish and these can be arbitrarily named, so we need to\n # query the API and figure out what triggers exist\n triggers = _get_triggers_for_action(server=server, action_identifier=identifier)\n if triggers is None:\n print(f\"\\nEncountered error getting triggers to publish for {identifier}!\")\n return False\n elif not triggers:\n print(f\"{identifier} has no triggers to publish - just skipping instead.\")\n # TODO: should we check this up-front and refuse to skip if there are no triggers that will be\n # published rather than carry on under the assumption that the user copy-pasted/typo'd the identifier?\n return True\n\n else:\n # TODO: this loop should use event_publish(), but we'd need to refactor how the CLI works and stop passing\n # around the full set of args everywhere to do so\n print(\"\\nTriggers to publish:\")\n print(\"\\n\".join(f\" * {trigger}\" for trigger in triggers) + \"\\n\")\n for trigger in triggers:\n print(f\"Publishing trigger {trigger}...\")\n if not request(\n url=urljoin(server, \"/api/events\"),\n data={\"command\": \"publish\", \"event\": trigger},\n ):\n print(\n f\"Failed to publish trigger {trigger} - you may want to retry this command or manually publish the trigger!\"\n )\n all_success = False\n else:\n print(f\"\\nFailed to skip {identifier}!\")\n return False\n\n return all_success\n\n\ndef control_objects(args: argparse.Namespace):\n tron_client = client.Client(args.server, user_attribution=True)\n url_index = tron_client.index()\n for identifier in args.id:\n try:\n tron_id = client.get_object_type_from_identifier(\n url_index,\n identifier,\n )\n except ValueError as e:\n possibilities = list(\n tron_jobs_completer(prefix=\"\", client=tron_client),\n )\n suggestions = suggest_possibilities(\n word=identifier,\n possibilities=possibilities,\n )\n raise SystemExit(f\"Error: {e}{suggestions}\")\n\n if args.command == \"skip-and-publish\":\n # this command is more of a pseudo-command - skip and publish are handled in two different resources\n # and changing the API would be painful, so instead we call skip + publish separately from the client\n # (i.e., this file) to implement this functionality\n yield skip_and_publish(args.server, tron_id, identifier)\n\n else:\n data = dict(command=args.command)\n if args.command == \"start\" and args.run_date:\n data[\"run_time\"] = str(args.run_date)\n yield request(urljoin(args.server, tron_id.url), data)\n # NOTE: ideally we'd add this message in the JobController handle_command() function, but having the API return terminal escape codes\n # sounds like a bad idea, so we're doing it here instead\n if args.command == \"disable\":\n print(\n warning_output(\n \"WARNING: jobs disabled with tronctl disable are *NOT* guaranteed to stay disabled. You must disable the job in yelpsoa-configs to guarantee it will not be re-enabled.\"\n )\n )\n\n\ndef retry(args):\n if args.deps_timeout != RetryAction.NO_TIMEOUT:\n deps_timeout_str = \"forever\" # timeout = -1 (RetryAction.WAIT_FOREVER)\n if args.deps_timeout > 0:\n deps_timeout_str = \"up to \" + str(datetime.timedelta(seconds=args.deps_timeout))\n print(\n f\"We will wait {deps_timeout_str} for all upstream triggers to be published \"\n \"and required actions to finish successfully before issuing retries for the \"\n \"following actions:\"\n )\n print()\n pprint.pprint(args.id)\n print()\n\n retries = retry_actions(args.server, args.id, args.use_latest_command, args.deps_timeout)\n print_retries_table(retries)\n yield all([r.succeeded for r in retries])\n\n\ndef move(args):\n try:\n old_name = args.id[0]\n new_name = args.id[1]\n except IndexError as e:\n raise SystemExit(f\"Error: Move command needs two arguments.\\n{e}\")\n\n tron_client = client.Client(args.server, user_attribution=True)\n url_index = tron_client.index()\n job_index = url_index[\"jobs\"]\n if old_name not in job_index.keys():\n raise SystemExit(f\"Error: {old_name} doesn't exist\")\n if new_name in job_index.keys():\n raise SystemExit(f\"Error: {new_name} exists already\")\n\n data = dict(command=\"move\", old_name=old_name, new_name=new_name)\n yield request(urljoin(args.server, \"/api/jobs\"), data)\n\n\ndef backfill(args):\n if not args.id:\n print(\"Error: must provide at least one id argument\")\n yield False\n if args.max_parallel > LIMIT_MAX_PARALLEL_RUNS:\n raise SystemExit(\n f\"The flag --max-parallel exceeds the allowed limit of {LIMIT_MAX_PARALLEL_RUNS}. \"\n + \"Please reach out to the Tron team if you need to run backfills with higher limits.\"\n )\n\n if args.start_date:\n if args.end_date is None:\n args.end_date = datetime.datetime.today()\n dates = get_date_range(args.start_date, args.end_date, descending=args.descending)\n else:\n dates = args.dates\n date_strs = [d.date().isoformat() for d in dates]\n\n job_name = args.id[0]\n if args.dry_run:\n print_backfill_cmds(job_name, date_strs)\n yield True\n else:\n if confirm_backfill(job_name, date_strs):\n loop = asyncio.get_event_loop()\n try:\n backfill_runs = loop.run_until_complete(\n run_backfill_for_date_range(\n args.server,\n job_name,\n dates,\n max_parallel=args.max_parallel,\n ignore_errors=(not args.fail_on_error),\n ),\n )\n finally:\n loop.close()\n\n print_backfill_runs_table(backfill_runs)\n yield all(br.run_state in BackfillRun.SUCCESS_STATES for br in backfill_runs)\n\n\ndef tron_version(args):\n local_version = __version__\n print(f\"Tron client version: {local_version}\")\n response = client.request(urljoin(args.server, \"/api/status\"))\n if response.error:\n print(f\"Error: {response.content}\")\n yield\n server_version = response.content.get(\"version\", \"unknown\")\n print(f\"Tron server version: {server_version}\")\n if server_version != local_version:\n print(\"Warning: client and server versions should match\")\n yield\n yield True\n\n\nCOMMANDS: dict[str, Callable[[argparse.Namespace], Generator[bool, None, None]]] = defaultdict(\n lambda: control_objects,\n publish=event_publish,\n discard=event_discard,\n backfill=backfill,\n move=move,\n retry=retry,\n version=tron_version,\n)\n\n\ndef main():\n \"\"\"run tronctl\"\"\"\n args = parse_cli()\n cmd_utils.load_config(args)\n\n # NOTE: we do this after load_configs() since load_config() may set some logging defaults that we want to override\n desired_level = cmd_utils.setup_logging(args)\n logging.getLogger().setLevel(desired_level)\n\n cmd = COMMANDS[args.command]\n try:\n for ret in cmd(args):\n if not ret:\n sys.exit(ExitCode.fail)\n except RequestError as err:\n print(\n f\"Error connecting to the tron server ({args.server}): {err}\",\n file=sys.stderr,\n )\n sys.exit(ExitCode.fail)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "bin/tronctl_tabcomplete.sh", "content": "if [[ -n ${ZSH_VERSION-} ]]; then\n\tautoload -U +X bashcompinit && bashcompinit\nfi\n\n# This magic eval enables tab-completion for tron commands\n# http://argcomplete.readthedocs.io/en/latest/index.html#synopsis\neval \"$(/opt/venvs/tron/bin/register-python-argcomplete tronctl)\"\n" }, { "path": "bin/trond", "content": "#!/usr/bin/env python\n\"\"\" Start the Tron server daemon.\"\"\"\nimport argparse\nimport faulthandler\nimport logging\nimport os\nimport time\nimport traceback\n\nimport pkg_resources\n\nimport tron\nfrom tron import trondaemon\nfrom tron.commands import cmd_utils\nfrom tron.config import manager\n\nlog = logging.getLogger(__name__)\n\nDEFAULT_CONF = \"default_config.yaml\"\nDEFAULT_CONF_PATH = \"config/\"\nDEFAULT_WORKING_DIR = \"/var/lib/tron/\"\nDEFAULT_LOCKFILE = \"tron.lock\"\nDEFAULT_LOCKPATH = \"/var/run/\" + DEFAULT_LOCKFILE\n\n\ndef parse_cli():\n parser = argparse.ArgumentParser()\n\n parser.add_argument(\n \"--version\",\n action=\"version\",\n version=f\"{parser.prog} {tron.__version__}\",\n )\n\n parser.add_argument(\n \"-w\",\n \"--working-dir\",\n default=DEFAULT_WORKING_DIR,\n help=\"Working directory for the Tron daemon, default %(default)s\",\n )\n\n parser.add_argument(\n \"-c\",\n \"--config-path\",\n default=DEFAULT_CONF_PATH,\n help=\"File path to the Tron configuration file\",\n )\n\n parser.add_argument(\n \"--nodaemon\",\n action=\"store_true\",\n default=False,\n help=\"[DEPRECATED] Disable daemonizing, default %(default)s\",\n )\n\n parser.add_argument( # for backwards compatibility\n \"--pid-file\",\n help=\"[DEPRECATED] File path to pid file. Use --lock-file instead.\",\n )\n\n parser.add_argument(\n \"--lock-file\",\n help=\"File path to lock file, defaults to %s if working directory \"\n \"is default. Otherwise defaults to /%s\" % (DEFAULT_LOCKPATH, DEFAULT_LOCKFILE),\n )\n\n logging_group = parser.add_argument_group(\"logging\", \"\")\n logging_group.add_argument(\n \"--log-conf\",\n \"-l\",\n help=\"File path to a custom logging.conf\",\n )\n\n logging_group.add_argument(\n \"-v\",\n \"--verbose\",\n action=\"count\",\n default=0,\n help=\"Verbose logging. Repeat for more verbosity.\",\n )\n\n logging_group.add_argument(\n \"--debug\",\n action=\"store_true\",\n help=\"Debug mode, extra error reporting, no daemonizing\",\n )\n\n api_group = parser.add_argument_group(\"Web Service API\", \"\")\n api_group.add_argument(\n \"--port\",\n \"-P\",\n dest=\"listen_port\",\n type=int,\n help=\"TCP port number to listen on, default %(default)s\",\n default=cmd_utils.DEFAULT_PORT,\n )\n\n api_group.add_argument(\n \"--host\",\n \"-H\",\n dest=\"listen_host\",\n help=\"Hostname to listen on, default %(default)s\",\n default=cmd_utils.DEFAULT_HOST,\n )\n\n requirement = pkg_resources.Requirement.parse(\"tron\")\n api_group.add_argument(\n \"--web-path\",\n default=pkg_resources.resource_filename(\n requirement,\n \"tronweb\",\n ),\n help=\"Path to static web resources, default %(default)s.\",\n )\n\n args = parser.parse_args()\n args.working_dir = os.path.abspath(args.working_dir)\n\n if args.log_conf:\n args.log_conf = os.path.join(args.working_dir, args.log_conf)\n if not os.path.exists(args.log_conf):\n parser.error(\"Logging config file not found: %s\" % args.log_conf)\n\n if not args.lock_file:\n if args.pid_file: # for backwards compatibility\n args.lock_file = args.pid_file\n elif args.working_dir == DEFAULT_WORKING_DIR:\n args.lock_file = DEFAULT_LOCKPATH\n else:\n args.lock_file = DEFAULT_LOCKFILE\n\n args.lock_file = os.path.join(args.working_dir, args.lock_file)\n args.config_path = os.path.join(\n args.working_dir,\n args.config_path,\n )\n\n return args\n\n\ndef create_default_config(config_path):\n \"\"\"Create a default empty configuration for first time installs\"\"\"\n default = pkg_resources.resource_string(tron.__name__, DEFAULT_CONF)\n manager.create_new_config(config_path, default)\n\n\ndef setup_environment(args):\n \"\"\"Setup the working directory and config environment.\"\"\"\n if not os.path.exists(args.working_dir):\n os.makedirs(args.working_dir)\n\n if not os.path.isdir(args.working_dir) or not os.access(args.working_dir, os.R_OK | os.W_OK | os.X_OK):\n msg = \"Error, can't access working directory %s\" % args.working_dir\n raise SystemExit(msg)\n\n # Attempt to create a default config if config is missing\n if not os.path.exists(args.config_path):\n try:\n create_default_config(args.config_path)\n except OSError as e:\n msg = \"Error creating default configuration at %s: %s\"\n log.debug(traceback.format_exc())\n raise SystemExit(msg % (args.config_path, e))\n\n if not os.access(args.config_path, os.R_OK | os.W_OK):\n msg = \"Error opening configuration %s: Missing Permissions\"\n raise SystemExit(msg % args.config_path)\n\n\ndef main():\n args = parse_cli()\n boot_time = time.time()\n setup_environment(args)\n trond = trondaemon.TronDaemon(args)\n trond.run(boot_time)\n\n\nif __name__ == \"__main__\":\n # print tracebacks on signals/faults\n # NOTE: you likely want to read https://docs.python.org/3/library/faulthandler.html\n # as these tracebacks will look slightly different\n faulthandler.enable()\n\n try:\n main()\n # this is a little weird, but every now and then we're seeing a mysterious tron exit\n # that doesn't seem to correspond with anything else - let's catch BaseException\n # (and therefore SystemExit) in case anything is calling sys.exit() since there's no\n # traceback when we see this\n except (\n BaseException,\n # technically, we really only need to catch BaseException - but let's be extra-paranoid\n Exception,\n ):\n traceback.print_exc()\n raise\n" }, { "path": "bin/tronfig", "content": "#!/usr/bin/env python\nimport logging\nimport os\nimport shutil\nimport sys\nimport tempfile\nimport traceback\n\nfrom tron.commands import cmd_utils\nfrom tron.commands.client import Client\nfrom tron.config import config_parse\nfrom tron.config import ConfigError\nfrom tron.config import manager\nfrom tron.config import schema\n\nlog = logging.getLogger(\"tronfig\")\n\n\ndef parse_cli():\n parser = cmd_utils.build_option_parser()\n\n parser.add_argument(\n \"-p\",\n \"--print\",\n action=\"store_true\",\n dest=\"print_config\",\n help=\"Print config to stdout, rather than uploading\",\n )\n parser.add_argument(\n \"-C\",\n \"--check\",\n action=\"store_true\",\n dest=\"check\",\n help=\"Upload and check configuration, don't apply, \"\n \"useful when you want to verify if tron daemon \"\n \"will accept your configuration.\",\n )\n parser.add_argument(\n \"-d\",\n \"--delete\",\n action=\"store_true\",\n help=\"Delete the configuration for this namespace\",\n )\n parser.add_argument(\n \"-V\",\n \"--validate\",\n action=\"store_true\",\n dest=\"validate\",\n help=\"Only validate configuration, don't upload, \"\n \"useful for verifying config locally. If namespace \"\n \"is not specified, it will be derived from file \"\n \"name, if any.\",\n )\n parser.add_argument(\n \"-D\",\n \"--validate-dir\",\n action=\"store_true\",\n dest=\"validate_dir\",\n help=\"Full validation of a folder, don't upload, \" \"same as -V but checks for more edge-cases\",\n )\n parser.add_argument(\n \"-n\",\n \"--namespace\",\n action=\"store\",\n help=\"Alternate namespace to use\",\n )\n parser.add_argument(\n \"-m\",\n \"--master-config\",\n action=\"store\",\n dest=\"master_config\",\n help=\"Source of master configuration file\",\n )\n parser.add_argument(\"source\")\n\n return parser.parse_args()\n\n\ndef upload_config(client, config_name, contents, config_hash, check=False):\n response = client.config(\n config_name,\n config_data=contents,\n config_hash=config_hash,\n check=check,\n )\n\n if \"error\" in response:\n log.error(response[\"error\"])\n return False\n\n print(\"Configuration uploaded successfully\", file=sys.stderr)\n return True\n\n\ndef validate(config_name, config_content, master_content=None):\n try:\n config_data = manager.from_string(config_content)\n master_data = (\n manager.from_string(\n master_content,\n )\n if master_content\n else None\n )\n config_parse.validate_fragment(\n name=config_name,\n fragment=config_data,\n master_config=master_data,\n )\n except ConfigError as e:\n return str(e)\n\n\ndef delete_config(client, config_name):\n if config_name == schema.MASTER_NAMESPACE:\n log.error(\n \"Deleting MASTER namespace is not allowed. Name must be specified.\",\n )\n return\n\n response = input(\n f\"This will delete the configuration for the {config_name} namespace. Proceed? (y/n): \",\n )\n if response[:1].lower() != \"y\":\n return\n\n config_hash = client.config(config_name)[\"hash\"]\n if upload_config(client, config_name, \"\", config_hash):\n return\n raise SystemExit(\"tronfig deletion failed\")\n\n\ndef validate_dir(path):\n try:\n manifest_dir = tempfile.mkdtemp()\n manifest = manager.ManifestFile(manifest_dir)\n manifest.create()\n for fname in os.listdir(path):\n name, ext = os.path.splitext(fname)\n if ext == \".yaml\":\n namespace = name\n manifest.add(namespace, os.path.join(path, fname))\n\n config_manager = manager.ConfigManager(path, manifest)\n config_manager.load()\n except ConfigError as e:\n traceback.print_exc()\n return str(e)\n finally:\n if manifest_dir:\n shutil.rmtree(manifest_dir)\n\n\ndef get_config_input(namespace, source):\n if source == \"-\":\n source_io = sys.stdin\n if not namespace:\n namespace = schema.MASTER_NAMESPACE\n else:\n source_io = open(source)\n if not namespace:\n name, _ = os.path.splitext(os.path.basename(source))\n namespace = name\n\n content = source_io.read()\n\n return namespace, content\n\n\nif __name__ == \"__main__\":\n args = parse_cli()\n cmd_utils.setup_logging(args)\n cmd_utils.load_config(args)\n\n if args.validate or args.validate_dir:\n if args.validate:\n name, content = get_config_input(args.namespace, args.source)\n master_content = None\n if args.master_config:\n _, master_content = get_config_input(\n schema.MASTER_NAMESPACE,\n args.master_config,\n )\n result = validate(\n config_name=name,\n config_content=content,\n master_content=master_content,\n )\n elif args.validate_dir:\n result = validate_dir(args.source)\n\n if not result:\n print(\"OK\")\n sys.exit(0)\n else:\n print(result)\n sys.exit(1)\n\n client = Client(args.server)\n\n if args.print_config:\n content = client.config(args.source)[\"config\"]\n if type(content) is not bytes:\n content = content.encode(\"utf8\")\n os.write(sys.stdout.fileno(), content)\n elif args.delete:\n delete_config(client, args.source)\n else:\n namespace, content = get_config_input(args.namespace, args.source)\n config_hash = client.config(namespace)[\"hash\"]\n result = validate(namespace, content)\n if result:\n print(result)\n sys.exit(1)\n\n if upload_config(\n client,\n namespace,\n content,\n config_hash,\n check=args.check,\n ):\n sys.exit(0)\n\n print(\"Uploading failed\")\n sys.exit(1)\n" }, { "path": "bin/tronrepl", "content": "#!/usr/bin/env python\n\"\"\" Start the Tron server daemon.\"\"\"\nimport argparse\nimport logging\nimport os\nimport traceback\n\nimport IPython\nimport pkg_resources\n\nimport tron.mcp\nfrom tron import trondaemon\nfrom tron.commands import cmd_utils\nfrom tron.config import manager\n\nlog = logging.getLogger(__name__)\n\nDEFAULT_CONF = \"default_config.yaml\"\nDEFAULT_CONF_PATH = \"config/\"\nDEFAULT_WORKING_DIR = \"/var/lib/tron/\"\nDEFAULT_LOCKFILE = \"tron-repl.lock\"\nDEFAULT_LOCKPATH = \"/var/run/\" + DEFAULT_LOCKFILE\n\n\ndef parse_cli():\n parser = argparse.ArgumentParser()\n\n parser.add_argument(\n \"--version\",\n action=\"version\",\n version=f\"{parser.prog} {tron.__version__}\",\n )\n\n parser.add_argument(\n \"-w\",\n \"--working-dir\",\n default=DEFAULT_WORKING_DIR,\n help=\"Working directory for the Tron daemon, default %(default)s\",\n )\n\n parser.add_argument(\n \"-c\",\n \"--config-path\",\n default=DEFAULT_CONF_PATH,\n help=\"File path to the Tron configuration file\",\n )\n\n parser.add_argument(\n \"--nodaemon\",\n action=\"store_true\",\n default=False,\n help=\"[DEPRECATED] Disable daemonizing, default %(default)s\",\n )\n\n parser.add_argument( # for backwards compatibility\n \"--pid-file\",\n help=\"[DEPRECATED] File path to pid file. Use --lock-file instead.\",\n )\n\n parser.add_argument(\n \"--lock-file\",\n help=\"File path to lock file, defaults to %s if working directory \"\n \"is default. Otherwise defaults to /%s\" % (DEFAULT_LOCKPATH, DEFAULT_LOCKFILE),\n )\n\n logging_group = parser.add_argument_group(\"logging\", \"\")\n logging_group.add_argument(\n \"--log-conf\",\n \"-l\",\n help=\"File path to a custom logging.conf\",\n )\n\n logging_group.add_argument(\n \"-v\",\n \"--verbose\",\n action=\"count\",\n default=0,\n help=\"Verbose logging. Repeat for more verbosity.\",\n )\n\n logging_group.add_argument(\n \"--debug\",\n action=\"store_true\",\n help=\"Debug mode, extra error reporting, no daemonizing\",\n )\n\n api_group = parser.add_argument_group(\"Web Service API\", \"\")\n api_group.add_argument(\n \"--port\",\n \"-P\",\n dest=\"listen_port\",\n type=int,\n help=\"TCP port number to listen on, default %(default)s\",\n default=cmd_utils.DEFAULT_PORT,\n )\n\n api_group.add_argument(\n \"--host\",\n \"-H\",\n dest=\"listen_host\",\n help=\"Hostname to listen on, default %(default)s\",\n default=cmd_utils.DEFAULT_HOST,\n )\n\n requirement = pkg_resources.Requirement.parse(\"tron\")\n api_group.add_argument(\n \"--web-path\",\n default=pkg_resources.resource_filename(\n requirement,\n \"tronweb\",\n ),\n help=\"Path to static web resources, default %(default)s.\",\n )\n\n args = parser.parse_args()\n args.working_dir = os.path.abspath(args.working_dir)\n\n if args.log_conf:\n args.log_conf = os.path.join(args.working_dir, args.log_conf)\n if not os.path.exists(args.log_conf):\n parser.error(\"Logging config file not found: %s\" % args.log_conf)\n\n if not args.lock_file:\n if args.pid_file: # for backwards compatibility\n args.lock_file = args.pid_file\n elif args.working_dir == DEFAULT_WORKING_DIR:\n args.lock_file = DEFAULT_LOCKPATH\n else:\n args.lock_file = DEFAULT_LOCKFILE\n\n args.lock_file = os.path.join(args.working_dir, args.lock_file)\n args.config_path = os.path.join(\n args.working_dir,\n args.config_path,\n )\n\n return args\n\n\ndef create_default_config(config_path):\n \"\"\"Create a default empty configuration for first time installs\"\"\"\n default = pkg_resources.resource_string(tron.__name__, DEFAULT_CONF)\n manager.create_new_config(config_path, default)\n\n\ndef setup_environment(args):\n \"\"\"Setup the working directory and config environment.\"\"\"\n if not os.path.exists(args.working_dir):\n os.makedirs(args.working_dir)\n\n if not os.path.isdir(args.working_dir) or not os.access(args.working_dir, os.R_OK | os.W_OK | os.X_OK):\n msg = \"Error, can't access working directory %s\" % args.working_dir\n raise SystemExit(msg)\n\n # Attempt to create a default config if config is missing\n if not os.path.exists(args.config_path):\n try:\n create_default_config(args.config_path)\n except OSError as e:\n msg = \"Error creating default configuration at %s: %s\"\n log.debug(traceback.format_exc())\n raise SystemExit(msg % (args.config_path, e))\n\n if not os.access(args.config_path, os.R_OK | os.W_OK):\n msg = \"Error opening configuration %s: Missing Permissions\"\n raise SystemExit(msg % args.config_path)\n\n\ndef main():\n args = parse_cli()\n\n setup_environment(args)\n trond = trondaemon.TronDaemon(args) # noqa: F841\n\n trond.mcp = tron.mcp.MasterControlProgram(\n trond.options.working_dir,\n trond.options.config_path,\n )\n trond.mcp._load_config()\n # trond.mcp.restore_state(trond.mcp.config.load().get_master().action_runner)\n\n # mcp = trond.mcp # noqa: F841\n # store = mcp.state_watcher.state_manager._impl # noqa: F841\n\n print(\"\")\n print(\"+---------------------+\")\n print(\"| Tron REPL |\")\n print(\"| Available locals: |\")\n print(\"| - trond |\")\n print(\"+---------------------+\")\n print(\"\")\n IPython.embed()\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "bin/tronview", "content": "#!/usr/bin/env python\nimport os\nimport sys\n\nimport argcomplete\n\nfrom tron.commands import cmd_utils\nfrom tron.commands import display\nfrom tron.commands.client import Client\nfrom tron.commands.client import get_object_type_from_identifier\nfrom tron.commands.client import RequestError\nfrom tron.commands.client import TronObjectType\nfrom tron.commands.cmd_utils import ExitCode\nfrom tron.commands.cmd_utils import suggest_possibilities\nfrom tron.commands.cmd_utils import tron_jobs_completer\n\n\ndef parse_cli():\n parser = cmd_utils.build_option_parser()\n parser.add_argument(\n \"--numshown\",\n \"-n\",\n type=int,\n dest=\"num_displays\",\n help=\"Max number of jobs/job-runs shown\",\n default=10,\n )\n parser.add_argument(\n \"--color\",\n \"-c\",\n action=\"store_true\",\n dest=\"display_color\",\n help=\"Display in color\",\n default=None,\n )\n parser.add_argument(\n \"--nocolor\",\n action=\"store_false\",\n dest=\"display_color\",\n help=\"Display without color\",\n default=None,\n )\n parser.add_argument(\n \"--stdout\",\n \"-o\",\n action=\"count\",\n dest=\"stdout\",\n help=\"Solely displays stdout\",\n default=0,\n )\n parser.add_argument(\n \"--stderr\",\n \"-e\",\n action=\"count\",\n dest=\"stderr\",\n help=\"Solely displays stderr\",\n default=0,\n )\n parser.add_argument(\n \"--events\",\n \"-E\",\n action=\"store_true\",\n dest=\"events\",\n help=\"Display stored events\",\n default=0,\n )\n parser.add_argument(\n \"name\",\n nargs=\"?\",\n help=\"job name | job run id | action id\",\n ).completer = cmd_utils.tron_jobs_completer\n\n argcomplete.autocomplete(parser)\n args = parser.parse_args()\n return args\n\n\ndef console_height():\n if not sys.stdout.isatty():\n return 40\n return int(os.popen(\"stty size\", \"r\").read().split()[0])\n\n\ndef view_all(args, client):\n \"\"\"Retrieve jobs and display them.\"\"\"\n return display.DisplayJobs().format(\n client.jobs(\n include_job_runs=False,\n include_action_runs=False,\n include_action_graph=False,\n include_node_pool=False,\n ),\n )\n\n\ndef view_job(args, job_id, client):\n \"\"\"Retrieve details of the specified job and display\"\"\"\n job_content = client.job(job_id.url, count=args.num_displays)\n return display.format_job_details(job_content)\n\n\ndef view_job_run(args, job_run_id, client):\n actions = client.job_runs(job_run_id.url)\n display_action = display.DisplayActionRuns()\n return display_action.format(actions)\n\n\ndef view_action_run(args, act_run_id, client):\n content = client.action_runs(\n act_run_id.url,\n num_lines=args.num_displays,\n )\n return display.format_action_run_details(content)\n\n\nobj_type_to_view_map = {\n TronObjectType.job: view_job,\n TronObjectType.job_run: view_job_run,\n TronObjectType.action_run: view_action_run,\n}\n\n\ndef get_view_output(name, args, client):\n url_index = client.index()\n try:\n tron_id = get_object_type_from_identifier(url_index, name)\n except ValueError as e:\n possibilities = list(tron_jobs_completer(prefix=\"\", client=client))\n suggestions = suggest_possibilities(\n word=name,\n possibilities=possibilities,\n )\n raise SystemExit(f\"Error: {e}{suggestions}\")\n\n if tron_id.type not in obj_type_to_view_map:\n return\n\n try:\n return obj_type_to_view_map[tron_id.type](args, tron_id, client)\n except RequestError as e:\n raise SystemExit(f\"Error: {e}\")\n\n\ndef main():\n \"\"\"run tronview\"\"\"\n args = parse_cli()\n cmd_utils.setup_logging(args)\n cmd_utils.load_config(args)\n\n display.Color.toggle(args.display_color)\n client = Client(args.server)\n\n try:\n if args.events:\n response = client.request(\"/api/events\")\n error = response.get(\"error\")\n if not error:\n for evt in response.get(\"response\", [\"* no recorded events *\"]):\n print(evt)\n sys.exit(ExitCode.success)\n\n if not args.name:\n output = view_all(args, client)\n else:\n output = get_view_output(args.name, args, client)\n\n if not output:\n print(\"Unrecognized identifier: %s\" % args.name, file=sys.stderr)\n sys.exit(ExitCode.fail)\n\n if sys.stdout.isatty() and len(output.split(\"\\n\")) > console_height():\n display.view_with_less(output, args.display_color)\n else:\n print(output)\n\n except RequestError as err:\n print(\n f\"Error connecting to the tron server ({args.server}): {err}\",\n file=sys.stderr,\n )\n sys.exit(ExitCode.fail)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "bin/tronview_tabcomplete.sh", "content": "if [[ -n ${ZSH_VERSION-} ]]; then\n\tautoload -U +X bashcompinit && bashcompinit\nfi\n\n# This magic eval enables tab-completion for tron commands\n# http://argcomplete.readthedocs.io/en/latest/index.html#synopsis\neval \"$(/opt/venvs/tron/bin/register-python-argcomplete tronview)\"\n" }, { "path": "contrib/migration_script.py", "content": "#!/usr/bin/env python\n\"\"\"\nThis script is for migrating jobs to another namespace\n\"\"\"\nimport argparse\nimport subprocess\nimport time\nfrom urllib.parse import urljoin\nfrom urllib.parse import urlparse\n\nfrom tron import yaml\nfrom tron.commands import client\n\n\nclass bcolors:\n HEADER = \"\\033[95m\"\n OKBLUE = \"\\033[94m\"\n OKGREEN = \"\\033[92m\"\n WARNING = \"\\033[93m\"\n FAIL = \"\\033[91m\"\n ENDC = \"\\033[0m\"\n BOLD = \"\\033[1m\"\n UNDERLINE = \"\\033[4m\"\n\n\ndef parse_args():\n parser = argparse.ArgumentParser(\n description=\"Migrate jobs to new namespace\",\n )\n parser.add_argument(\n \"--server\",\n required=True,\n help=\"specify the location of tron master\",\n )\n parser.add_argument(\n \"--old-ns\",\n required=True,\n help=\"Old namespace\",\n )\n parser.add_argument(\n \"--new-ns\",\n required=True,\n help=\"New namespace\",\n )\n parser.add_argument(\n \"source\",\n help=\"source file to get list of jobs\",\n )\n parser.add_argument(\n \"--job\",\n help=\"Specify a single job to migrate\",\n )\n args = parser.parse_args()\n return args\n\n\ndef check_job_if_running(jobs_status, job_name):\n for job_status in jobs_status:\n if job_status[\"name\"] == job_name:\n status = job_status[\"status\"]\n if status == \"running\":\n print(bcolors.FAIL + f\"job {job_name} is still running, can not migrate\" + bcolors.ENDC)\n return False\n elif status == \"disabled\":\n print(bcolors.WARNING + f\"job {job_name} is disabled, need to cancel it manually later\" + bcolors.ENDC)\n return True\n else:\n print(bcolors.OKGREEN + f\"job {job_name} is not running, can migrate\" + bcolors.ENDC)\n return True\n\n print(bcolors.FAIL + f\"Can not find the job {job_name}\" + bcolors.ENDC)\n return False\n\n\ndef command_jobs(command, jobs, args, ns=None):\n \"\"\"This function run tronctl command for the jobs\n command: the tronctl command it will run\n jobs: a list of jobs\n args: the args for this script\n ns: the namespace to use as the prefix for each job, if None, the scrip would use args.old_ns instead\n \"\"\"\n data = {\"command\": command}\n command_flag = True\n for job in jobs:\n if ns is not None:\n job_name = ns + \".\" + job[\"name\"]\n else:\n job_name = args.old_ns + \".\" + job[\"name\"]\n\n if command == \"move\":\n data = {\n \"command\": command,\n \"old_name\": args.old_ns + \".\" + job[\"name\"],\n \"new_name\": args.new_ns + \".\" + job[\"name\"],\n }\n uri = urljoin(args.server, \"api/jobs\")\n job_name = args.new_ns + \".\" + job[\"name\"]\n else:\n data = {\"command\": command}\n uri = urljoin(args.server, \"api/jobs/\" + job_name)\n\n response = client.request(uri, data=data)\n if response.error:\n print(bcolors.FAIL + f\"Failed to {command} {job_name}\" + bcolors.ENDC)\n command_flag = False\n else:\n print(bcolors.OKGREEN + f\"Succeed to {command} {job_name}\" + bcolors.ENDC)\n return command_flag\n\n\ndef ssh_command(hostname, command):\n print(bcolors.BOLD + f\"Executing the command: ssh -A {hostname} {command}\" + bcolors.ENDC)\n ssh = subprocess.Popen(\n [\"ssh\", \"-A\", hostname, command],\n shell=False,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n )\n exitcode = ssh.wait()\n result = ssh.stdout.readlines()\n error = ssh.stderr.readlines()\n if exitcode != 0:\n print(bcolors.FAIL + f\"Execute command {command} failed: {error}\" + bcolors.ENDC)\n exit(exitcode)\n return result\n\n\ndef main():\n args = parse_args()\n filename = args.source\n hostname = urlparse(args.server).hostname\n if filename.endswith(\".yaml\"):\n tron_client = client.Client(args.server)\n jobs_status = tron_client.jobs()\n\n is_migration_safe = True\n with open(filename) as f:\n jobs = yaml.load(f)[\"jobs\"]\n job_names = [job[\"name\"] for job in jobs]\n if args.job is not None: # only want to migrate specific job\n # Overwrite existing jobs since only migrating one job\n jobs = [job for job in jobs if job[\"name\"] == args.job]\n if not jobs:\n raise ValueError(f\"Invalid job specified. Options were {job_names}\")\n job_name_with_ns = args.old_ns + \".\" + args.job\n is_migration_safe = is_migration_safe & check_job_if_running(jobs_status, job_name_with_ns)\n\n else: # Migrate all jobs in namespace\n for job_name in job_names:\n job_name_with_ns = args.old_ns + \".\" + job_name\n is_migration_safe = is_migration_safe & check_job_if_running(jobs_status, job_name_with_ns)\n\n if is_migration_safe is True:\n print(bcolors.OKBLUE + \"Jobs are not running.\" + bcolors.ENDC)\n else:\n print(bcolors.WARNING + \"Some jobs are still running, abort this migration,\" + bcolors.ENDC)\n return\n\n # try stop cron\n ssh_command(hostname, \"sudo service cron stop\")\n\n # wait unitil yelpsoa-configs branch is merged\n res = input(\"Merge and push yelpsoa-configs branch. Ready to continue? [y/n]\")\n if res == \"y\":\n # wait for 10 seconds after pushing the branch\n time.sleep(30)\n # rsyn yelpsoa-configs\n command = \"sudo rsync -a --delay-updates --contimeout=10 --timeout=10 --chmod=Du+rwx,go+rx --port=8731 --delete yelpsoa-slave.local.yelpcorp.com::yelpsoa-configs /nail/etc/services\"\n ssh_command(hostname, command)\n\n # migrate jobs to new namespace\n command_jobs(\"move\", jobs, args)\n\n # update new namespace\n ssh_command(hostname, \"sudo paasta_setup_tron_namespace \" + args.new_ns)\n\n # update old namespace if only one job is moving\n if args.job:\n ssh_command(hostname, \"sudo paasta_setup_tron_namespace \" + args.old_ns)\n\n # clean up namespace\n ssh_command(hostname, \"sudo paasta_cleanup_tron_namespaces\")\n\n # start cron\n ssh_command(hostname, \"sudo service cron start\")\n\n return\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "contrib/mock_patch_checker.py", "content": "#!/usr/bin/env python3.10\nimport ast\nimport sys\n\n\nclass MockChecker(ast.NodeVisitor):\n def __init__(self):\n self.errors = 0\n self.init_module_imports()\n\n def init_module_imports(self):\n self.imported_patch = False\n self.imported_mock = False\n\n def check_files(self, files):\n for file in files:\n self.check_file(file)\n\n def check_file(self, filename):\n self.current_filename = filename\n try:\n with open(filename) as fd:\n try:\n file_ast = ast.parse(fd.read())\n except SyntaxError as error:\n print(\"SyntaxError on file %s:%d\" % (filename, error.lineno))\n return\n except OSError:\n print(\"Error opening filename: %s\" % filename)\n return\n self.init_module_imports()\n self.visit(file_ast)\n\n def _call_uses_patch(self, node):\n try:\n return node.func.id == \"patch\"\n except AttributeError:\n return False\n\n def _call_uses_mock_patch(self, node):\n try:\n return node.func.value.id == \"mock\" and node.func.attr == \"patch\"\n except AttributeError:\n return False\n\n def visit_Import(self, node):\n if [name for name in node.names if \"mock\" == name.name]:\n self.imported_mock = True\n\n def visit_ImportFrom(self, node):\n if node.module == \"mock\" and (name for name in node.names if \"patch\" == name.name):\n self.imported_patch = True\n\n def visit_Call(self, node):\n try:\n if (self.imported_patch and self._call_uses_patch(node)) or (\n self.imported_mock and self._call_uses_mock_patch(node)\n ):\n if not any([keyword for keyword in node.keywords if keyword.arg == \"autospec\"]):\n print(\"%s:%d: Found a mock without an autospec!\" % (self.current_filename, node.lineno))\n self.errors += 1\n except AttributeError:\n pass\n self.generic_visit(node)\n\n\ndef main(filenames):\n checker = MockChecker()\n checker.check_files(filenames)\n if checker.errors == 0:\n sys.exit(0)\n else:\n print(\"You probably meant to specify 'autospec=True' in these tests.\")\n print(\"If you really don't want to, specify 'autospec=None'\")\n sys.exit(1)\n\n\nif __name__ == \"__main__\":\n main(sys.argv[1:])\n" }, { "path": "contrib/namespace_cleanup.sh", "content": "#/bin/bash\n\necosystem=\"stagef\"\n\nread -p \"Are you at tron-$ecosystem (y/n)?\" RES\necho\nif [ $RES = \"y\" ]; then\n #load namespace from _manifest.yaml\n for namespace in $(cat /nail/tron/config/_manifest.yaml | uq | jq -r 'keys[]')\n do\n file=$(cat /nail/tron/config/_manifest.yaml | uq | jq -r .\\\"$namespace\\\")\n filename=$(basename $file)\n if [ -f \"/nail/etc/services/tron/$ecosystem/$filename\" ]; then\n echo \"$namespace is up to date\"\n elif [ $namespace == \"MASTER\" ]; then\n echo \"It is MASTER namepsace\"\n else\n num_job=$(cat /nail/tron/config/$filename | uq | jq -r \".jobs | length\")\n echo \"========= $filename =========\"\n cat /nail/tron/config/$filename\n echo \"=============================\"\n if [ $num_job == 0 ]; then\n echo \"$namespace is left behind, deleting the namespace\"\n tronfig -d $namespace\n else\n echo \"Can't remove the namespace since it is not empty.\"\n fi\n fi\n done\nelse\n echo \"Please change the ecosystem variable in this script or execute this script at tron-$ecosystem\"\nfi\n" }, { "path": "contrib/patch-config-loggers.diff", "content": "--- a/debian/tron/opt/venvs/tron/lib/python3.10/site-packages/kubernetes/client/configuration.py\n+++ b/debian/tron/opt/venvs/tron/lib/python3.10/site-packages/kubernetes/client/configuration.py\n@@ -71,11 +71,11 @@\n \"\"\"\n\n _default = None\n-\n def __init__(self, host=\"http://localhost\",\n api_key=None, api_key_prefix=None,\n username=None, password=None,\n discard_unknown_keys=False,\n+ is_logger_used=False,\n ):\n \"\"\"Constructor\n \"\"\"\n@@ -106,26 +106,28 @@\n \"\"\"Password for HTTP basic authentication\n \"\"\"\n self.discard_unknown_keys = discard_unknown_keys\n+ self.is_logger_used = is_logger_used\n self.logger = {}\n- \"\"\"Logging Settings\n- \"\"\"\n- self.logger[\"package_logger\"] = logging.getLogger(\"client\")\n- self.logger[\"urllib3_logger\"] = logging.getLogger(\"urllib3\")\n- self.logger_format = '%(asctime)s %(levelname)s %(message)s'\n- \"\"\"Log format\n- \"\"\"\n- self.logger_stream_handler = None\n- \"\"\"Log stream handler\n- \"\"\"\n- self.logger_file_handler = None\n- \"\"\"Log file handler\n- \"\"\"\n- self.logger_file = None\n- \"\"\"Debug file location\n- \"\"\"\n- self.debug = False\n- \"\"\"Debug switch\n- \"\"\"\n+ if self.is_logger_used:\n+ \"\"\"Logging Settings\n+ \"\"\"\n+ self.logger[\"package_logger\"] = logging.getLogger(\"client\")\n+ self.logger[\"urllib3_logger\"] = logging.getLogger(\"urllib3\")\n+ self.logger_format = '%(asctime)s %(levelname)s %(message)s'\n+ \"\"\"Log format\n+ \"\"\"\n+ self.logger_stream_handler = None\n+ \"\"\"Log stream handler\n+ \"\"\"\n+ self.logger_file_handler = None\n+ \"\"\"Log file handler\n+ \"\"\"\n+ self.logger_file = None\n+ \"\"\"Debug file location\n+ \"\"\"\n+ self.debug = False\n+ \"\"\"Debug switch\n+ \"\"\"\n\n self.verify_ssl = True\n \"\"\"SSL/TLS verification\n@@ -178,11 +180,12 @@\n for k, v in self.__dict__.items():\n if k not in ('logger', 'logger_file_handler'):\n setattr(result, k, copy.deepcopy(v, memo))\n- # shallow copy of loggers\n- result.logger = copy.copy(self.logger)\n- # use setters to configure loggers\n- result.logger_file = self.logger_file\n- result.debug = self.debug\n+ if self.is_logger_used:\n+ # shallow copy of loggers\n+ result.logger = copy.copy(self.logger)\n+ # use setters to configure loggers\n+ result.logger_file = self.logger_file\n+ result.debug = self.debug\n return result\n\n @classmethod\n" }, { "path": "contrib/sync-from-yelp-prod.sh", "content": "#!/bin/bash\nrsync --exclude=.stderr --exclude=.stdout -aPv tron-prod:/nail/tron/* example-cluster/\ngit checkout example-cluster/logging.conf\n\necho \"\"\necho \"Now Run:\"\necho \"\"\necho \" tox -e example-cluster\"\necho \" ./example-cluster/start.sh\"\n" }, { "path": "contrib/sync_namespaces_jobs.py", "content": "#!/usr/bin/env python\n\"\"\" This script is for load testing of Tron\n\nHistorically, Tronview and Tronweb were (are) slow. To better understand the performance\nbottleneck of Tron, we could use this script to generate the fake namespaces and\njobs as many as we want to perform load testing. Ticket TRON-70 tracks the progress\nof speeding up Tronview and Tronweb.\n\"\"\"\nimport argparse\nimport os\n\nfrom tron import yaml\n\n\ndef parse_args():\n parser = argparse.ArgumentParser(\n description=\"Creating namespaces and jobs configuration for load testing\",\n )\n parser.add_argument(\n \"--multiple\",\n type=int,\n default=1,\n help=\"multiple workload of namespaces and jobs from source directory\",\n )\n parser.add_argument(\n \"--src\",\n default=\"/nail/etc/services/tron/prod\",\n help=\"Directory to get Tron configuration files\",\n )\n parser.add_argument(\n \"--dest\",\n default=\"/tmp/tron-servdir\",\n help=\"Directory to put Tron configuration files for load testing\",\n )\n args = parser.parse_args()\n return args\n\n\ndef main():\n args = parse_args()\n for filename in os.listdir(args.src):\n print(f\"filename = {filename}\")\n filepath = os.path.join(args.src, filename)\n if os.path.isfile(filepath) and filepath.endswith(\".yaml\"):\n with open(filepath) as f:\n config = yaml.load(f)\n\n if filename == \"MASTER.yaml\":\n for key in list(config):\n if key != \"jobs\":\n del config[key]\n\n jobs = config.get(\"jobs\", [])\n if jobs is not None:\n for job in jobs:\n job[\"node\"] = \"localhost\"\n if \"monitoring\" in job:\n del job[\"monitoring\"]\n for action in job.get(\"actions\", []):\n action[\"command\"] = \"sleep 10s\"\n if \"node\" in action:\n action[\"node\"] = \"localhost\"\n for i in range(args.multiple):\n out_filepath = os.path.join(\n args.dest,\n \"load_testing_\" + str(i) + \"-\" + filename,\n )\n with open(out_filepath, \"w\") as outf:\n yaml.dump(config, outf, default_flow_style=False)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "debian/changelog", "content": "tron (3.10.0) jammy; urgency=medium\n\n * 3.10.0 tagged with 'make release'\n Commit: Merge pull request #1096 from Yelp/u/kkasp/unlearn-how-to-\n read-pickles U/kkasp/unlearn how to read pickles\n\n -- Kevin Kaspari Mon, 16 Mar 2026 09:03:19 -0700\n\ntron (3.9.3) jammy; urgency=medium\n\n * 3.9.3 tagged with 'make release'\n Commit: Merge pull request #1098 from jhereth/u/jhereth/MLCOMPUTE-\n 6270/add-attempt-number-label Add k8s label for attempt number\n\n -- Kevin Kaspari Mon, 16 Mar 2026 07:40:42 -0700\n\ntron (3.9.2) jammy; urgency=medium\n\n * 3.9.2 tagged with 'make release'\n Commit: Merge pull request #1090 from Yelp/u/kkasp/TRON-2452-\n compress-json Compress json\n\n -- Kevin Kaspari Mon, 09 Feb 2026 13:24:43 -0800\n\ntron (3.9.1) jammy; urgency=medium\n\n * 3.9.1 tagged with 'make release'\n Commit: Merge pull request #1084 from Yelp/jfong/TRON-2546-fix-spot-\n terminatioons-not-ooms TRON-2546: Fix spot termination\n identification in current k8s version\n\n -- Jen Patague Thu, 08 Jan 2026 09:58:32 -0800\n\ntron (3.9.0) jammy; urgency=medium\n\n * 3.9.0 tagged with 'make release'\n Commit: Upgrade Tron to Python 3.10 - TRON-2435 (#1071) * Upgrade\n Tron to Python 3.10 * Upgrade zope.interface to a compatible python\n 3.10 version * Moved moto to requirements dev instead and removed\n mock from minimal * lowering object_size for more room * Update\n classifier and path in itest\n\n -- Eman Elsabban Thu, 18 Dec 2025 06:24:50 -0800\n\ntron (3.8.9) jammy; urgency=medium\n\n * 3.8.9 tagged with 'make release'\n Commit: Assume we\"ll only see yaml files in logreader.py (#1077) If\n someone is using a .yml file - they\"ve already gone horribly off-\n track :p\n\n -- Luis Perez Mon, 24 Nov 2025 13:42:34 -0800\n\ntron (3.8.8) jammy; urgency=medium\n\n * 3.8.8 tagged with 'make release'\n Commit: Try to warn folks about incorrect `tronctl publish` usage\n (#1076) It\"s absurdly easy to use this incorrectly (especially if\n you\"ve been woken up by a page ;p) - so let\"s try to warn folks if\n they\"re potentially using `tronctl publish` incorrectly afaik, the\n usual mistakes here are to use a job/action id (detectable by the\n run number) OR to use a too long/short trigger id (either from copy-\n pasting too much or too little) I\"ve opted to not error out when\n this happens since I\"m somewhat sure that it\"s possible to manually\n construct triggers that don\"t match this format and depend on those\n triggers in soaconfigs. If anyone with more time wants to vibe-code\n something to verify this (and maybe even add some validation to the\n tronfig schema ;)), then we can always error out immediately\n\n -- Luis Perez Thu, 20 Nov 2025 12:01:45 -0800\n\ntron (3.8.7) jammy; urgency=medium\n\n * 3.8.7 tagged with 'make release'\n Commit: Lightly refactor skip-and-publish (and add some more output)\n (#1075) The current output is pretty minimal and doesn\"t help us\n (or users) debug particularly efficiently whenever something isn\"t\n working as expected. Output now looks something like: ``` Skipping\n compute-infra-test-service.always_fail.5.fail... ActionRun: compute-\n infra-test-service.always_fail.5.fail now in state skipped\n Successfully skipped compute-infra-test-service.always_fail.5.fail.\n Fetching triggers to publish for compute-infra-test-\n service.always_fail.5.fail... Triggers to publish: * compute-\n infra-test-service.always_fail.fail.shortdate.2025-11-15 Publishing\n trigger compute-infra-test-service.always_fail.fail.shortdate.2025-\n 11-15... OK ``` I\"ve lightly refactored this since I don\"t really\n know why I inlined all this logic in 2021 and it was bothering me.\n\n -- Luis Perez Wed, 12 Nov 2025 06:46:31 -0800\n\ntron (3.8.6) jammy; urgency=medium\n\n * 3.8.6 tagged with 'make release'\n Commit: Add another disk eviction substring (#1074) We\"re also\n seeing messages that look like: `Pod ephemeral local storage usage\n exceeds the total limit of containers 1Gi.` which don\"t trigger the\n existing check. This adds \"ephemeral local storage\" as another\n trigger without removing the existing one. I\"m not quite sure how\n k8s decides which message to use without reading through the k8s\n code further, but it appears that both cases are possible:\n https://github.com/kubernetes/kubernetes/blob/350481c4747da5c2ad4f24\n e71e7edaabbd4cfe2e/pkg/kubelet/eviction/helpers.go#L53-L57\n\n -- Luis Perez Mon, 03 Nov 2025 09:56:04 -0800\n\ntron (3.8.5) jammy; urgency=medium\n\n * 3.8.5 tagged with 'make release'\n Commit: TRON-2480: Fix tron systemd unit to retry indefinitely & add\n minor delay between restarts (#1070) We saw in\n https://yelp.slack.com/archives/CA53K7S68/p1755111528654579 and\n https://yelp.slack.com/archives/CA53K7S68/p1752625385958589?thread_t\n s=1752624525.373959&cid=CA53K7S68 my change from #1066 to add the\n `ExecStartPre` , combined with the default `RestartSec=0` and\n `StartLimitBurst=5` meant that tron in a cluster that required some\n time to perform the shutdown process would still see `trond` still\n running and immediately run through the 5 allowed attempts, then\n fail to restart unless retriggered by a timer or socket: > Note\n that units which are configured for Restart=, and which reach the\n start limit are not attempted to be restarted anymore; however, they\n may still be restarted manually or from a timer or socket at a later\n point, after the interval has passed. This PR now disables the\n restart limits entirely by setting both `StartLimitBurst` and\n `StartLimitIntervalSec` to 0, as per the docs: > interval is a time\n span with the default unit of seconds, but other units may be\n specified, see\n [systemd.time(7)](https://www.freedesktop.org/software/systemd/man/l\n atest/systemd.time.html#). The special value \"infinity\" can be used\n to limit the total number of start attempts, even if they happen at\n large time intervals. Defaults to DefaultStartLimitIntervalSec= in\n manager configuration file, and may be set to 0 to disable any kind\n of rate limiting. (Note this indicates only setting\n `StartLimitIntervalSec=0` is necessary to disable the rate limiting,\n but chatgpt convinced me it makes it clearer to the reader if both\n are set to 0, happy to undo this if we think we\"ll be confused\n later) These settings seem to work on tron-infrastage after\n modifying it to cause an artificial 60s delay when shutting down, as\n we can see restarts are delayed 10s between the next attempt, and\n eventually tron is able to start cleanly on its own with no user\n intervention:\n https://fluffy.yelpcorp.com/i/qnk5kPRNXsMPRbpLv32KvKDLFk1gl72p.html\n\n -- Luis Perez Mon, 18 Aug 2025 09:22:27 -0700\n\ntron (3.8.4) jammy; urgency=medium\n\n * 3.8.4 tagged with 'make release'\n Commit: Default sliders to show 50 items (#1067) Defaulting to 10\n doesn\"t actually save any time since we\"ve fetched all the data\n anyway. Let\"s instead default to 50 since that should should be the\n most common run_limit This will make pages longer, but it will be\n less confusing to folks when they get alerts for failed/unknown/etc\n jobs that are past the visible runs with the previous slider\n configuration.\n\n -- Luis Perez Wed, 23 Jul 2025 14:10:18 -0700\n\ntron (3.8.3) jammy; urgency=medium\n\n * 3.8.3 tagged with 'make release'\n Commit: Merge pull request #1066 from Yelp/jfong/TRON-2340-systemd-\n sigterm-handling TRON-2340: Update tron unit file to ensure proper\n sigterm handling\n\n -- Jen Patague Mon, 14 Jul 2025 13:51:48 -0700\n\ntron (3.8.2) jammy; urgency=medium\n\n * 3.8.2 tagged with 'make release'\n Commit: Released 3.8.1 via make release\n\n -- Jen Patague Mon, 14 Jul 2025 12:42:31 -0700\n\ntron (3.8.1) jammy; urgency=medium\n\n * 3.8.1 tagged with 'make release'\n Commit: Enable disallow_untyped_decorators mypy option (#1064) For\n once, a small mypy PR :p There was just a single untyped decorator\n and I went ahead and wrote some inline comments \"cause I don\"t think\n I\"ll remember how to read this after this gets merged :deepfried-\n loljpg:\n\n -- Luis Perez Wed, 02 Jul 2025 10:14:27 -0700\n\ntron (3.8.0) jammy; urgency=medium\n\n * 3.8.0 tagged with 'make release'\n Commit: Allow transitioning to success/fail from any terminal state\n (#1057) This is helpful when folks have handled failures outside of\n Tron (or have done something outside of Tron that nominally means\n that the Tron status is now incorrect) OR to work-around any\n weirdness with Tron states (e.g., something funky happened in k8s\n and Tron never saw the event) NOTE: this PR changes a bunch of\n extra files - but that\"s just due to mypy following more code paths\n after I typed the ActionRunController `__init__` so that I could go-\n to-def in my editor :p Additionally, kkasp and I have occasionally\n been runing into an state where `check-requirements` says a\n mismatched version of `cryptography` is installed - I finally got\n around to debugging this and realized this stems from our very hacky\n extra_requirements_yelp.txt shenanigans: pyopenssl has lower/upper-\n bounds on cryptography but, due to the ordering we install things\n in, `check-requirements` might run before we\"ve installed pyopenssl\n and downgraded cryptography (and pass) or after (and fail) The real\n changes are confined to\n `tron/core/action.py::ActionRun::STATE_MACHINE` (and a now-deleted\n ActionRun tests that are no longer valid)\n\n -- Luis Perez Mon, 30 Jun 2025 10:16:53 -0700\n\ntron (3.7.3) jammy; urgency=medium\n\n * 3.7.3 tagged with 'make release'\n Commit: more gracefully handle errors during backfill monitoring\n (#1062)\n\n -- Matteo Piano Thu, 26 Jun 2025 05:54:02 -0700\n\ntron (3.7.2) jammy; urgency=medium\n\n * 3.7.2 tagged with 'make release'\n Commit: Merge pull request #1056 from Yelp/u/kkasp/rest-in-peace-\n tronweb2 Delete tronweb2\n\n -- Kevin Kaspari Wed, 25 Jun 2025 10:25:34 -0700\n\ntron (3.7.1) jammy; urgency=medium\n\n * 3.7.1 tagged with 'make release'\n Commit: Merge pull request #1055 from Yelp/u/kkasp/TRON-2007-tron-\n disk-evictions Add disk eviction error code and check logic\n\n -- Kevin Kaspari Fri, 20 Jun 2025 10:11:01 -0700\n\ntron (3.7.0) jammy; urgency=medium\n\n * 3.7.0 tagged with 'make release'\n Commit: Save idempotent to json - TRON-2154 (#1051) * Save\n idempotent to json * set idempotent to false if it doesnt exist *\n address review\n\n -- Eman Elsabban Wed, 18 Jun 2025 13:09:02 -0700\n\ntron (3.6.3) jammy; urgency=medium\n\n * 3.6.3 tagged with 'make release'\n Commit: ignore cached okta token in backfills (#1054)\n\n -- Matteo Piano Mon, 16 Jun 2025 01:38:02 -0700\n\ntron (3.6.2) jammy; urgency=medium\n\n * 3.6.2 tagged with 'make release'\n Commit: cache auth tokens before starting backfill jobs (#1053)\n\n -- Matteo Piano Fri, 13 Jun 2025 01:23:13 -0700\n\ntron (3.6.1) jammy; urgency=medium\n\n * 3.6.1 tagged with 'make release'\n Commit: return auth error that clients can parse (#1052)\n\n -- Matteo Piano Mon, 09 Jun 2025 05:24:25 -0700\n\ntron (3.6.0) jammy; urgency=medium\n\n * 3.6.0 tagged with 'make release'\n Commit: Merge pull request #1046 from Yelp/u/kkasp/TRON-2391-\n aggregate-metrics U/kkasp/tron 2391 aggregate metrics\n\n -- Kevin Kaspari Wed, 04 Jun 2025 10:43:44 -0700\n\ntron (3.5.2) jammy; urgency=medium\n\n * 3.5.2 tagged with 'make release'\n Commit: use oidc token method from vault-tools (#1050)\n\n -- Matteo Piano Wed, 28 May 2025 07:36:00 -0700\n\ntron (3.5.1) jammy; urgency=medium\n\n * 3.5.1 tagged with 'make release'\n Commit: fix some annoying warnings (#1049)\n\n -- Matteo Piano Tue, 27 May 2025 10:22:44 -0700\n\ntron (3.5.0) jammy; urgency=medium\n\n * 3.5.0 tagged with 'make release'\n Commit: add support for api auth via vault (#1048) add support for\n api auth via vault\n\n -- Matteo Piano Tue, 27 May 2025 08:47:48 -0700\n\ntron (3.4.10) jammy; urgency=medium\n\n * 3.4.10 tagged with 'make release'\n Commit: Merge pull request #1045 from Yelp/u/kkasp/TRON-2414-fix-\n broken-config-parse U/kkasp/tron 2414 fix broken config parse\n\n -- Kevin Kaspari Thu, 08 May 2025 09:51:28 -0700\n\ntron (3.4.9) jammy; urgency=medium\n\n * 3.4.9 tagged with 'make release'\n Commit: Merge pull request #1043 from Yelp/u/kkasp/TRON-2414-set-\n extra-safe-transaction-limit Drop under the limit a bit more for\n maximum safety\n\n -- Kevin Kaspari Thu, 01 May 2025 10:54:54 -0700\n\ntron (3.4.8) jammy; urgency=medium\n\n * 3.4.8 tagged with 'make release'\n Commit: fix build_url_request when method is None (#1041)\n\n -- Luis Perez Mon, 14 Apr 2025 10:52:51 -0700\n\ntron (3.4.7) jammy; urgency=medium\n\n * 3.4.7 tagged with 'make release'\n Commit: Merge pull request #1042 from Yelp/u/kkasp/DAR-2637-lower-\n max-transact-items Reduce transact size to avoid cap\n\n -- Kevin Kaspari Thu, 10 Apr 2025 08:24:26 -0700\n\ntron (3.4.6) jammy; urgency=medium\n\n * 3.4.6 tagged with 'make release'\n Commit: Merge pull request #1039 from Yelp/u/kkasp/KKASP-0001-\n download-more-time Upgrade moment and get the latest tz info\n\n -- Kevin Kaspari Mon, 07 Apr 2025 11:10:20 -0700\n\ntron (3.4.5) jammy; urgency=medium\n\n * 3.4.5 tagged with 'make release'\n Commit: Bump task_processing from 1.3.4 to 1.3.5 (#1040) I made a\n silly mistake and didn\"t increase the attempt counter\n\n -- Luis Perez Thu, 03 Apr 2025 15:37:22 -0700\n\ntron (3.4.4) jammy; urgency=medium\n\n * 3.4.4 tagged with 'make release'\n Commit: Update task_processing to 1.3.4 for watch backoff (#1038)\n This includes https://github.com/Yelp/task_processing/pull/225,\n which should add some backoff to watch restarts to avoid slamming\n the apiserver\n\n -- Luis Perez Thu, 03 Apr 2025 14:01:31 -0700\n\ntron (3.4.3) jammy; urgency=medium\n\n * 3.4.3 tagged with 'make release'\n Commit: Bump kubernetes clientlib to the latest supported version\n (#1037) See https://github.com/kubernetes-client/python?tab=readme-\n ov-file#compatibility\n\n -- Luis Perez Thu, 03 Apr 2025 10:46:44 -0700\n\ntron (3.4.2) jammy; urgency=medium\n\n * 3.4.2 tagged with 'make release'\n Commit: Merge pull request #1036 from Yelp/u/kkasp/TRON-2396-lower-\n min-zoom-for-big-ol-charts Halve minZoom for some of our big charts\n\n -- Kevin Kaspari Thu, 27 Mar 2025 10:42:14 -0700\n\ntron (3.4.1) jammy; urgency=medium\n\n * 3.4.1 tagged with 'make release'\n Commit: Bump task_processing to v1.3.3 for Watch restart fix (#1035)\n This brings in https://github.com/Yelp/task_processing/pull/223,\n which should ensure that we don\"t get stuck in a funky restart loop\n if k8s tells us our watch\"s resourceVersion is too old\n\n -- Luis Perez Wed, 26 Mar 2025 11:46:03 -0700\n\ntron (3.4.0) jammy; urgency=medium\n\n * 3.4.0 tagged with 'make release'\n Commit: Merge pull request #1032 from Yelp/u/kkasp/TRON-2370-do-ya-\n like-dags U/kkasp/tron 2370 do ya like dags\n\n -- Kevin Kaspari Fri, 21 Mar 2025 13:17:02 -0700\n\ntron (3.3.2) jammy; urgency=medium\n\n * 3.3.2 tagged with 'make release'\n Commit: Merge pull request #1034 from Yelp/u/mpiano/SEC-\n 19862_fix_service_param fix service name extraction logic\n\n -- Kevin Kaspari Fri, 21 Mar 2025 11:28:33 -0700\n\ntron (3.3.1) jammy; urgency=medium\n\n * 3.3.1 tagged with 'make release'\n Commit: Merge pull request #1033 from Yelp/u/mpiano/SEC-19862_fix\n fix auth middleware integration\n\n -- Kevin Kaspari Mon, 17 Mar 2025 12:47:36 -0700\n\ntron (3.3.0) jammy; urgency=medium\n\n * 3.3.0 tagged with 'make release'\n Commit: Merge pull request #1005 from Yelp/u/mpiano/SEC-19555 auth\n support for Tron APIs\n\n -- Kevin Kaspari Thu, 13 Mar 2025 11:30:42 -0700\n\ntron (3.2.12) jammy; urgency=medium\n\n * 3.2.12 tagged with 'make release'\n Commit: Merge pull request #1030 from Yelp/jfong/TRON-1797-tronweb-\n path TRON-1797: Put tronweb in a python-version-agnostic location\n\n -- Jen Patague Mon, 03 Mar 2025 12:15:00 -0800\n\ntron (3.2.11) jammy; urgency=medium\n\n * 3.2.11 tagged with 'make release'\n Commit: Try to load config_name_mapping from disk just once (#1013)\n There\"s enough files (and enough YAML in these files) that the\n IO/YAML parsing takes a significant amount of time. While it\"s nice\n to always load from the source-of-truth (i.e., the files on-disk) -\n it\"s not worth paying the performance penalty (especially at the\n scale we\"re seeing internally) for a negligible benefit. Locally\n (with a large test config), this results in a ~5x improvement in\n timings. More concretely, my test configs were taking ~30ish seconds\n *without* this diff and ~6ish seconds *with* this diff.\n\n -- Luis Perez Mon, 24 Feb 2025 09:08:41 -0800\n\ntron (3.2.10) jammy; urgency=medium\n\n * 3.2.10 tagged with 'make release'\n Commit: Merge pull request #1028 from Yelp/jfong/TRON-2333 TRON-\n 2333: Bump task_processing for pod truncation fix\n\n -- Jen Patague Fri, 21 Feb 2025 11:08:55 -0800\n\ntron (3.2.9) jammy; urgency=medium\n\n * 3.2.9 tagged with 'make release'\n Commit: Merge pull request #1023 from Yelp/u/kkasp/TRON-2342-\n exponential-backoff-dynamo-get Add dynamodb retry config for\n throttling and other errors. Add exponential backoff and jitter for\n unprocessed keys. Fix edge case where we succesfully process keys on\n our last attempt but still fail\n\n -- Kevin Kaspari Wed, 12 Feb 2025 13:03:36 -0800\n\ntron (3.2.8) jammy; urgency=medium\n\n * 3.2.8 tagged with 'make release'\n Commit: Merge pull request #1025 from Yelp/u/kkasp/DAR-2558-fix-\n overstep Fix loop boundary on getting partitions\n\n -- Kevin Kaspari Thu, 30 Jan 2025 11:59:51 -0800\n\ntron (3.2.7) jammy; urgency=medium\n\n * 3.2.7 tagged with 'make release'\n Commit: Merge pull request #1027 from Yelp/u/emanelsabban/TRON-2354\n Handling timezone aware jobs\n\n -- Eman Elsabban Fri, 24 Jan 2025 12:38:21 -0800\n\ntron (3.2.6) jammy; urgency=medium\n\n * 3.2.6 tagged with 'make release'\n Commit: Add default behaviour for jobs that dont have some keys and\n handle the non-existent of some json_vals (#1017) * Add default\n behaviour for jobs that dont have some keys and handle the non-\n existent of some json_vals * Remove get from command * cap add and\n drop should have values * removing the try/except since it was\n added in a separate pr * maybe not all gets are necessary * Adding\n comments\n\n -- Eman Elsabban Fri, 17 Jan 2025 11:47:48 -0800\n\ntron (3.2.5) jammy; urgency=medium\n\n * 3.2.5 tagged with 'make release'\n Commit: Merge pull request #1026 from Yelp/u/kkasp/fix-timezones\n Use localize instead of replace when writing tz info\n\n -- Kevin Kaspari Fri, 17 Jan 2025 11:00:55 -0800\n\ntron (3.2.4) jammy; urgency=medium\n\n * 3.2.4 tagged with 'make release'\n Commit: Merge pull request #1024 from Yelp/u/kkasp/empty-val Add\n check for empty val. Merge json correctly\n\n -- Kevin Kaspari Thu, 16 Jan 2025 08:49:56 -0800\n\ntron (3.2.3) jammy; urgency=medium\n\n * 3.2.3 tagged with 'make release'\n Commit: Merge pull request #1016 from Yelp/u/kkasp/TRON-2239-migrate-\n pickle-only-state-to-JSON U/kkasp/tron 2239 migrate pickle only\n state to json\n\n -- Kevin Kaspari Tue, 14 Jan 2025 12:05:21 -0800\n\ntron (3.2.2) jammy; urgency=medium\n\n * 3.2.2 tagged with 'make release'\n Commit: Merge pull request #1014 from Yelp/u/kkasp/fix-tooltip-lol-\n its-bootstrap-2.3.1 Disable the tooltip animation that is causing a\n strange problem where you can\"t get a tooltip when entering from the\n top of an object\n\n -- Kevin Kaspari Mon, 13 Jan 2025 06:59:56 -0800\n\ntron (3.2.1) jammy; urgency=medium\n\n * 3.2.1 tagged with 'make release'\n Commit: Merge pull request #1022 from Yelp/u/kkasp/pass-json-back-to-\n save-queue Pass json_val back to the save queue on failure\n\n -- Kevin Kaspari Fri, 10 Jan 2025 07:27:17 -0800\n\ntron (3.2.0) jammy; urgency=medium\n\n * 3.2.0 tagged with 'make release'\n Commit: Merge pull request #1018 from Yelp/u/kkasp/update-delete-\n item Update delete_item logic to handle json partitions\n\n -- Kevin Kaspari Thu, 09 Jan 2025 12:29:29 -0800\n\ntron (3.1.2) jammy; urgency=medium\n\n * 3.1.2 tagged with 'make release'\n Commit: Bump task_processing to v1.3.1 for capability fix (#1020)\n Just like Yelp/paasta#3972 and Yelp/paasta#3973, we need to ensure\n that there are no duplicates between cap_add and cap_drop -\n otherwise, the cap_drop entry will \"win\" and the duplicate\n capability will not be added.\n\n -- Luis Perez Thu, 09 Jan 2025 09:21:33 -0800\n\ntron (3.1.1) jammy; urgency=medium\n\n * 3.1.1 tagged with 'make release'\n Commit: Merge pull request #1019 from Yelp/u/emanelsabban/capture-\n json_val Capturing exception when json_val doesnt exist\n\n -- Eman Elsabban Wed, 08 Jan 2025 13:04:07 -0800\n\ntron (3.1.0) jammy; urgency=medium\n\n * 3.1.0 tagged with 'make release'\n Commit: Add read_json to be a part of master.yaml config (#1015) *\n Add read_json to be a part of master.yaml config * Fix the mypy\n error & other failed tests * Addressing reviews * Add reas_json to\n restore in shelve function\n\n -- Eman Elsabban Thu, 02 Jan 2025 12:51:55 -0800\n\ntron (3.0.0) jammy; urgency=medium\n\n * 3.0.0 tagged with 'make release'\n Commit: [TRON-2238] Reading from JSON to restore jobs\" state instead\n of pickles (#1010) * Reading from Json to restore jobs\" state\n instead of pickles * Fixing some bugs through testing * Deleting\n some comments * mocking get_config_watcher * Try mocking one more\n time * Toggling off read_json since we want to merge that way *\n Removing some comments * Addressed reviews * Toggling read_json\n back off * Addressing more reviews * one more review plz *\n includes reviews except tz\n\n -- Eman Elsabban Mon, 23 Dec 2024 07:56:01 -0800\n\ntron (2.8.1) jammy; urgency=medium\n\n * 2.8.1 tagged with 'make release'\n Commit: Merge pull request #1004 from Yelp/u/kkasp/TRON-2237-\n timedeltas-cant-json Use total_seconds for timedeltas. Log job and\n jobrun names/runnums on serialization\n\n -- Kevin Kaspari Thu, 31 Oct 2024 09:11:35 -0700\n\ntron (2.8.0) jammy; urgency=medium\n\n * 2.8.0 tagged with 'make release'\n Commit: Merge pull request #997 from Yelp/u/kkasp/TRON-2237-write-\n json-state Write JSON state\n\n -- Kevin Kaspari Tue, 29 Oct 2024 14:14:38 -0700\n\ntron (2.7.0) jammy; urgency=medium\n\n * 2.7.0 tagged with 'make release'\n Commit: Merge pull request #1001 from\n Yelp/u/cuza/getting_namespace_logs_from_yelpsoa Fixing Tron logs\n for jobs using other services images\n\n -- Kevin Kaspari Tue, 29 Oct 2024 13:31:16 -0700\n\ntron (2.6.0) jammy; urgency=medium\n\n * 2.6.0 tagged with 'make release'\n Commit: Use logreader rathern than vector-reader (#1002) This CLI\n is being renamed, so let\"s account for that\n\n -- Luis Perez Thu, 24 Oct 2024 09:34:05 -0700\n\ntron (2.5.3) jammy; urgency=medium\n\n * 2.5.3 tagged with 'make release'\n Commit: Merge pull request #1000 from Yelp/jfong/TRON-2208-\n nonretryable-to-unknown TRON-2208: Update non_retryable_exit_code\n behavior to treat as UNKNOWN\n\n -- Jen Patague Mon, 30 Sep 2024 14:12:37 -0700\n\ntron (2.5.2) jammy; urgency=medium\n\n * 2.5.2 tagged with 'make release'\n Commit: Use vector-reader cli if new logging pipeline is enabled\n (#999) Thankfully, this was a pretty easy change - we really only\n needed to parametrize the command name being used and the location\n selector is now a lot simpler as it\"ll always be the superregion\n\n -- Luis Perez Mon, 30 Sep 2024 09:32:22 -0700\n\ntron (2.5.1) jammy; urgency=medium\n\n * 2.5.1 tagged with 'make release'\n Commit: Merge pull request #998 from Yelp/jfong/TRON-2277-fix-\n nonretryable-exit-codes TRON-2277: Pass along\n non_retryable_exit_codes to KubernetesCluster objects\n\n -- Jen Patague Thu, 26 Sep 2024 10:21:53 -0700\n\ntron (2.5.0) jammy; urgency=medium\n\n * 2.5.0 tagged with 'make release'\n Commit: Attempt to batch config loading for deployments (#996)\n Right now we make at most 2N calls to the Tron API during config\n deployments: N to get the current configs and at most N if all\n services have changes. To start, I\"d like to reduce this to N by\n allowing GET /api/config to return all the configs so that the only\n requests needed are POSTs for changed configs. Depending on how\n this goes, we can look into batching up the POSTs so that we can\n also do that in a single request. In terms of speed, it looks like\n loading all the configs from pnw-prod (on my devbox) with this new\n behavior takes ~3s - which isn\"t great, but there\"s a decent bit of\n file IO going on here :(\n\n -- Luis Perez Tue, 17 Sep 2024 08:25:15 -0700\n\ntron (2.4.2) jammy; urgency=medium\n\n * 2.4.2 tagged with 'make release'\n Commit: Merge pull request #995 from Yelp/u/cuza/making-non-\n retryable-exit-code-accept-negative-numbers Fix negative value\n check for non-retryable exit codes in Kubernetes configuration\n\n -- Dave Cuza Mon, 09 Sep 2024 12:23:49 -0700\n\ntron (2.4.1) jammy; urgency=medium\n\n * 2.4.1 tagged with 'make release'\n Commit: Merge pull request #994 from Yelp/StefanoChiodino-patch-1\n Update tronweb.less\n\n -- Luis Perez Tue, 03 Sep 2024 11:24:45 -0700\n\ntron (2.4.0) jammy; urgency=medium\n\n * 2.4.0 tagged with 'make release'\n Commit: TRON-2208: Add toggle in tron config to disable retries on\n LOST k8s jobs (#988) Given that \"LOST\" means Tron has lost track of\n a pod it already thought it had started for a job, attempting to\n retry/start a replacement can be dangerous for non-idempotent jobs.\n In the current state, these will consume retries, but with some of\n our EKS migration methods, LOST tasks are more likely. Therefore, we\n should have a way to temporarily pause retries on these. ## Related\n Issues - TRON-2208: Add toggle in Tron config to disable retries on\n LOST k8s jobs\n\n -- Luis Perez Wed, 28 Aug 2024 09:19:14 -0700\n\ntron (2.3.0) jammy; urgency=medium\n\n * 2.3.0 tagged with 'make release'\n Commit: Merge pull request #989 from Yelp/jfong/TRON-2195-old-\n kubeconfig-paths TRON-2195: Support watcher_kubeconfig_paths\n\n -- Jen Patague Thu, 11 Jul 2024 14:25:42 -0700\n\ntron (2.2.7) jammy; urgency=medium\n\n * 2.2.7 tagged with 'make release'\n Commit: Only show disable warning on tronctl disable (#990) I\"m not\n sure I was thinking here since this ended up unconditionally\n printing the disable warning - but we can chalk this up to an\n intentional PR campaign to warn folks and only show the warnings on\n tronctl disable from now on :p\n\n -- Luis Perez Thu, 11 Jul 2024 11:49:18 -0700\n\ntron (2.2.6) jammy; urgency=medium\n\n * 2.2.6 tagged with 'make release'\n Commit: Update scribereader tests (#983)\n\n -- Yaroslav Liakhovskyi Fri, 28 Jun 2024 01:41:29 -0700\n\ntron (2.2.5) jammy; urgency=medium\n\n * 2.2.5 tagged with 'make release'\n Commit: Update yelp_clog and use datetime range for S3 logs (#979)\n\n -- Yaroslav Liakhovskyi Wed, 26 Jun 2024 08:27:10 -0700\n\ntron (2.2.4) jammy; urgency=medium\n\n * 2.2.4 tagged with 'make release'\n Commit: Merge pull request #981 from Yelp/revert-980-\n u/jfong/revert_to_2.1.1 Revert \"Reverts all changes back through\n 2.1.1 and retains urgent unprocessedkeys fix\"\n\n -- Eman Elsabban Thu, 20 Jun 2024 14:52:09 -0700\n\ntron (2.2.3) jammy; urgency=medium\n\n * 2.2.3 tagged with 'make release'\n Commit: Reverts all changes back through 2.1.1 and retains urgent\n unprocessedkeys fix (#980) * Revert \"pass around projected SA\n configs properly\" This reverts commit\n 12b1bf27e7b9e9f3fd0963f9d073d2552a58115f. * Revert \"Merge remote-\n tracking branch \"origin/u/mpiano/SEC-18955\"\" This reverts commit\n ea6376d72ffdd269e11cb9338d4f0c656bcd6f66, reversing changes made to\n 4038f1e173aeff932b6e060fc2690f7aa502a85d. * Revert \"Use\n S3LogsReader with superregion and UTC timezone (#972)\" This reverts\n commit af73799363549a082eea11f0f98d5c7f4810abe4.\n\n -- Eman Elsabban Thu, 20 Jun 2024 10:07:42 -0700\n\ntron (2.2.2) jammy; urgency=medium\n\n * 2.2.2 tagged with 'make release'\n Commit: Merge pull request #977 from Yelp/u/jfong/tron_quick_fix\n Fix UnprocessedKeys bug when restoring from dynamodb\n\n -- Jen Patague Mon, 17 Jun 2024 17:50:59 -0700\n\ntron (2.2.1) jammy; urgency=medium\n\n * 2.2.1 tagged with 'make release'\n Commit: Merge pull request #976 from Yelp/u/mpiano/SEC-18955_fix\n pass around projected SA configs properly\n\n -- Jen Patague Mon, 17 Jun 2024 14:52:24 -0700\n\ntron (2.2.0) jammy; urgency=medium\n\n * 2.2.0 tagged with 'make release'\n Commit: Merge remote-tracking branch \"origin/u/mpiano/SEC-18955\"\n\n -- Matteo Piano Tue, 11 Jun 2024 02:27:25 -0700\n\ntron (2.1.2) jammy; urgency=medium\n\n * 2.1.2 tagged with 'make release'\n Commit: Use S3LogsReader with superregion and UTC timezone (#972)\n\n -- Yaroslav Liakhovskyi Wed, 05 Jun 2024 02:17:04 -0700\n\ntron (2.1.1) jammy; urgency=medium\n\n * 2.1.1 tagged with 'make release'\n Commit: Handling exceptions thrown from threads span by\n ThreadPoolExecutor - TRON-2202 (#969) * Handling exceptions thrown\n from threads span by ThreadPoolExecutor * Addressing reviews *\n Addressing more reviews * Removed Exit code\n\n -- Eman Elsabban Tue, 04 Jun 2024 11:24:00 -0700\n\ntron (2.1.0) jammy; urgency=medium\n\n * 2.1.0 tagged with 'make release'\n Commit: Revert the Mesos code deletions (#970) This turned out to\n be unsafe to to our use of pickles as a serialization format. *\n Revert \"Use the latest task_proc (#966)\" This reverts commit\n 01003a980854bc25ed2764c880e1fb69db296fb1. * Revert \"Delete\n remaining Mesos code (#961)\" This reverts commit\n 1f71d0fa406e530ac943f1fcaf312224015f392c. * Revert \"Delete Mesos\n related exit codes and docker files - TRON-2187 (#959)\" This\n reverts commit 33ad2a1657aeea61f899380dada42825e137083b. * Revert\n \"Delete Mesos logging config (#962)\" This reverts commit\n 640362424f1b1b6bb1c959a04b0843ca1e846c10. * Revert \"Merge pull\n request #953 from Yelp/u/emanelsabban/TRON-2183\" This reverts\n commit 10353457221d11f2a587c665bae05b16f3cba447, reversing changes\n made to e4114088fefaf6a3d3f5be27e5caf41d7c1c9973. * Revert\n \"Deleting Mesos code from the Master Control Program\" This reverts\n commit 18f48ee1db23e1e7f91e0e9e7846eb345d171534.\n\n -- Luis Perez Mon, 03 Jun 2024 13:33:05 -0700\n\ntron (2.0.0) jammy; urgency=medium\n\n * 2.0.0 tagged with 'make release'\n Commit: Use the latest task_proc (#966) There\"s no real changes\n here other than dropping all the Mesos-related code from task_proc\n\n -- Luis Perez Fri, 17 May 2024 13:49:55 -0700\n\ntron (1.32.5) jammy; urgency=medium\n\n * 1.32.5 tagged with 'make release'\n Commit: Merge pull request #965 from Yelp/u/emanelsabban/fix-make-\n release Remove /docs/source/generated from gitignore file\n\n -- Eman Elsabban Fri, 17 May 2024 09:10:54 -0700\n\ntron (1.32.4) jammy; urgency=medium\n\n * 1.32.4 tagged with 'make release'\n Commit: Delete Mesos related exit codes and docker files - TRON-2187\n (#959) * Delete Mesos related exit codes and docker files * Adding\n a try/except for validating extra keys * Testing deleted extra keys\n\n -- Eman Elsabban Fri, 17 May 2024 07:51:23 -0700\n\ntron (1.32.3) jammy; urgency=medium\n\n * 1.32.3 tagged with 'make release'\n Commit: Automate make release in Tron (#960)\n\n -- Jon Lee Mon, 13 May 2024 15:36:34 -0700\n\ntron (1.32.2) jammy; urgency=medium\n\n * 1.32.2 tagged with 'make release'\n Commit: Merge pull request #953 from Yelp/u/emanelsabban/TRON-2183\n Delete Mesos code from statemanager - TRON-2183\n\n -- Eman Elsabban Mon, 13 May 2024 12:00:06 -0700\n\ntron (1.32.1) jammy; urgency=medium\n\n * 1.32.1 tagged with 'make release'\n Commit: Merge pull request #952 from Yelp/u/emanelsabban/TRON-2182\n Deleting Mesos code from the Master Control Program - TRON-2182\n\n -- Eman Elsabban Mon, 13 May 2024 11:43:05 -0700\n\ntron (1.32.0) jammy; urgency=medium\n\n * 1.32.0 tagged with 'make release'\n Commit: Update task_proc to better handle killing tasks (#951)\n We've seen that task_proc/k8s will sometimes not correctly send\n events for pods that we try to kill ourselves (either because the\n pods are already gone or because the event is somehow missing data),\n so this task_proc version will send synthetic events when we call\n kill() to ensure that tron is in the correct state :)\n Co-authored-by: Jen Patague \n\n -- Luis Perez Thu, 09 May 2024 09:16:24 -0700\n\ntron (1.31.0) jammy; urgency=medium\n\n * 1.31.0 tagged with 'make release'\n Commit: Parallelizing execution of restoring in Tron - TRON-2161\n (#950) * Parallelizing execution of restoring in Tron * Addressing\n reviews and fixing unit tests * Adding sorting for runs state\n otherwise runs will be out of order * Addressing more reviews and\n adding exception for exceeding max_attempts * Address typing stuff\n and timers * Fixing the typing error * fixing incomplete sentence\n\n -- Eman Elsabban Wed, 01 May 2024 12:54:10 -0700\n\ntron (1.30.0) jammy; urgency=medium\n\n * 1.30.0 tagged with 'make release'\n Commit: Adding yelp_clog S3LogsReader (#949) * Upgrade yelp\n scribereader deps * Enable S3LogsReader for action run logs *\n Upgrade boto requirements * Upgrade mypy and update type ignores *\n Pin more internal requirements\n\n -- Yaroslav Liakhovskyi Mon, 29 Apr 2024 04:57:37 -0700\n\ntron (1.29.5) jammy; urgency=medium\n\n * 1.29.5 tagged with 'make release'\n Commit: Merge pull request #940 from Yelp/jfong/TRON-1850-starting-\n jobs-stuck TRON-1850: Include 'starting' pods in check for stuck jobs\n\n -- Jen Patague Tue, 09 Apr 2024 11:37:15 -0700\n\ntron (1.29.4) jammy; urgency=medium\n\n * 1.29.4 tagged with 'make release'\n Commit: Adding logs to Tron to indicate start and scheduling times -\n TRON-2152 (#948) * Adding logs to Tron to indicate start and\n scheduling times * Addressed reviews on the PR * Address reviews\n 2.0 * Deleting the start_schedule_jobs flag * Making boot_time a\n required param * bring back time.time in duration\n\n -- Eman Elsabban Mon, 08 Apr 2024 11:15:48 -0700\n\ntron (1.29.3) jammy; urgency=medium\n\n * 1.29.3 tagged with 'make release'\n Commit: Add a top-level exception handler (#947) As the comment\n says: let's see if adding a top-level handler (that then re-raises)\n and catching BaseException will give us more info as to what's\n sometimes causing Tron to exit. I've also added the usual base\n exception (Exception) just to be extra-safe (h/t to krall for the\n idea)\n\n -- Luis Perez Wed, 27 Mar 2024 15:00:00 -0700\n\ntron (1.29.2) jammy; urgency=medium\n\n * 1.29.2 tagged with 'make release'\n Commit: Merge pull request #946 from Yelp/u/emanelsabban/edit-\n messages Editing some of the messages from previous prs\n\n -- Eman Elsabban Wed, 20 Mar 2024 08:33:45 -0700\n\ntron (1.29.1) jammy; urgency=medium\n\n * 1.29.1 tagged with 'make release'\n Commit: Merge pull request #942 from Yelp/u/kkasp/TRON-1970-escape-\n html-tronweb Use Underscores HTML escaped interpolation in template\n for log and command outputs\n\n -- Kevin Kaspari Fri, 08 Mar 2024 08:03:14 -0800\n\ntron (1.29.0) jammy; urgency=medium\n\n * 1.29.0 tagged with 'make release'\n Commit: Merge pull request #944 from Yelp/u/emanelsabban/TRON-2124-\n expose-prom-metrics Adding prometheus endpoint in tron - TRON-2124\n\n -- Eman Elsabban Tue, 05 Mar 2024 12:55:14 -0800\n\ntron (1.28.5) jammy; urgency=medium\n\n * 1.28.5 tagged with 'make release'\n Commit: Stop keeping old copies of eventbus files (#941) We've\n never restored from these 'backups', and as long as we ensure that\n the current file points to a fully written file, it's fine to only\n keep at most 2 files around: the actual current file and a temporary\n 'new' file. To ensure that we're not pointing at a half-written\n file, we continue using the age-old pattern for atomic file updates:\n write to another file and swap a symlink once the write is complete\n\n -- Luis Perez Tue, 20 Feb 2024 11:55:45 -0800\n\ntron (1.28.4) jammy; urgency=medium\n\n * 1.28.4 tagged with 'make release'\n Commit: Merge pull request #939 from Yelp/u/wilmerrafael/COMPINFRA-\n 3601_adding_tron_run_number_label Adding tron run id to pod labels\n for k8s\n\n -- Wilmer Bandres Thu, 15 Feb 2024 06:43:40 -0800\n\ntron (1.28.3) jammy; urgency=medium\n\n * 1.28.3 tagged with 'make release'\n Commit: Merge pull request #938 from Yelp/u/kkasp/TRON-2112-lock-\n start Add lock to tron start to mitigate the risk of running\n duplicate jobs…\n\n -- Kevin Kaspari Thu, 08 Feb 2024 11:24:39 -0800\n\ntron (1.28.2) jammy; urgency=medium\n\n * 1.28.2 tagged with 'make release'\n Commit: Remove unnecessary mocks (#937) This is what I get for\n trusting GHA rather than also running the tests internally :) These\n mocks are no longer required and are actually causing test failures\n internally.\n\n -- Luis Perez Thu, 01 Feb 2024 09:10:42 -0800\n\ntron (1.28.1) jammy; urgency=medium\n\n * 1.28.1 tagged with 'make release'\n Commit: Downpin yelp-clog (#936) We had bumped this a couple major\n versions since we were also bumping scribereader - but we reverted\n the scribereader bump before merging the jammy/py38 branch and\n forgot to also revert the yelp-clog bump :)\n\n -- Luis Perez Thu, 01 Feb 2024 08:51:40 -0800\n\ntron (1.28.0) jammy; urgency=medium\n\n * 1.28.0 tagged with 'make release'\n Commit: Upgrading Tron to py3.8 + patching it with the fix (#934) *\n Monkeypatch SimpleQueue back to PySimpleQueue We have a hunch that\n this is what is causing our pod event loop to have wildly delayed\n items * Revert 'Revert python/jammy upgrades (#907)' This reverts\n commit 483da5c0fb258b01b8e47912ad034d43554ada7d. * new formatting\n * Bump pyyaml * Rm sad test that is not relevant for validation *\n added stuff to run tron locally * matching the python version with\n whats currently running in infrastage * adding also changelog *\n This commit includes some requriements for clog and try/except block\n for handle_events * This commit adds the patch fix * Revert\n 'matching the python version with whats currently running in\n infrastage' This reverts commit\n 1f81a6d4805d742a7a7a28b1d7d8eef39522a896. * Revert 'adding also\n changelog' This reverts commit\n 951fcc8fc31114bde340bad4b82ced0529e03b65. * precommit fixes the\n patch * Fixing mypy issues and tests failing * removing return and\n adding comment for handling defer being none * addressing wording\n * fix InvariantException back to Exception --------- Co-authored-\n by: Luis Perez Co-authored-by: Vincent Thibault\n \n\n -- Eman Elsabban Wed, 31 Jan 2024 09:25:13 -0800\n\ntron (1.27.5) jammy; urgency=medium\n\n * 1.27.5 tagged with 'make release'\n Commit: Allow Tron to use more than 10k file descriptors (#923)\n There appears to be an fd leak somewhere in tron, we're not in\n danger of hitting any host-level limits - so let's update the unit\n file to have a bigger limit in the meantime\n\n -- Luis Perez Wed, 11 Oct 2023 12:11:22 -0700\n\ntron (1.27.4) jammy; urgency=medium\n\n * 1.27.4 tagged with 'make release'\n Commit: Revert 'Link tronweb_url ' (#928) Reverts #911 as it did\n not actually result in a clickable link in Slack\n\n -- Luis Perez Tue, 10 Oct 2023 12:26:07 -0700\n\ntron (1.27.3) jammy; urgency=medium\n\n * 1.27.3 tagged with 'make release'\n Commit: Catch all exceptions in k8s submit_command() (#926) It's\n entirely possible that creating a task_processing task (and/or\n submitting one) can result in an exception. At the moment, this\n results in the affected ActionRun getting stuck in the Starting\n state - but this is a lie and means that the normal\n monitoring/alerting on failed runs does not kick in.\n\n -- Luis Perez Mon, 21 Aug 2023 12:27:26 -0700\n\ntron (1.27.2) jammy; urgency=medium\n\n * 1.27.2 tagged with 'make release'\n Commit: Merge pull request #924 from Yelp/u/vit/fix-\n configsecretvolume Fix item assignment issue with\n ConfigSecretVolume\n\n -- Vincent Thibault Mon, 14 Aug 2023 13:36:37 -0700\n\ntron (1.27.1) jammy; urgency=medium\n\n * 1.27.1 tagged with 'make release'\n Commit: Update to the latest task_proc (#915) This version pulls in\n a fix that stops pods from being launched with a null request. This\n has been running fine in infrastage where I've verified that Pods\n are indeed being launched with the correct metadata I've verified\n this as well with unit tests in task_proc :)\n\n -- Luis Perez Mon, 10 Jul 2023 08:28:15 -0700\n\ntron (1.27.0) jammy; urgency=medium\n\n * 1.27.0 tagged with 'make release'\n Commit: Remove tronweb2 code (#914) This isn't currently being used\n and is complicating the internal build process\n\n -- Luis Perez Tue, 13 Jun 2023 08:49:02 -0700\n\ntron (1.26.0) jammy; urgency=medium\n\n * 1.26.0 tagged with 'make release'\n Commit: Merge pull request #909 from Yelp/u/vit/tron-secret-volume\n TRON-1636: Pass secret_volumes to taskproc from kubernetes\n\n -- Vincent Thibault Mon, 12 Jun 2023 09:03:39 -0700\n\ntron (1.25.1) jammy; urgency=low\n\n * 1.25.1 tagged with 'make release'\n Commit: Merge pull request #911 from Yelp/tchen/link-tronweb-url\n Link tronweb_url\n\n -- Tianle Chen Mon, 05 Jun 2023 09:21:09 -0700\n\ntron (1.25.0) jammy; urgency=medium\n\n * 1.25.0 tagged with 'make release'\n Commit: COMPINFRA-2565: enforce an upper limit on backfill\n concurrency (#906) Problem ----- The Kubernetes control plane can\n be overwhelmed if a high number of concurrent backfills are\n executed. We currently set a relatively high default of and enforce\n no upper limit at all. Solution ----- * Reduce the default from\n to to reduce the baseline pressure of Tron backfill jobs. *\n Introduce and enforce a hard limit of concurrent backfills per\n tronctl invocation In a future PR, we might want to track the total\n count of running backfills per user in Zookeeper but this change\n should already safeguard against overwhelming Tron with backfills in\n the meantime. Signed-off-by: Max Falk \n\n -- Luis Perez Thu, 18 May 2023 09:06:24 -0700\n\ntron (1.24.5) jammy; urgency=medium\n\n * 1.24.5 tagged with 'make release'\n Commit: Revert python/jammy upgrades (#907) * Revert 'Upgrades\n requirements and adds caching on get loggers (#900)' This reverts\n commit 0ac2b69d648bb3b915a80637a4f3051e06a16296. * Revert 'Adds\n Python 3.8; build for Jammy (#896)' This reverts commit\n f7679b9a281d026b4d956c5735875eded5602310.\n\n -- Luis Perez Wed, 17 May 2023 08:44:16 -0700\n\ntron (1.24.4) jammy; urgency=medium\n\n * 1.24.4 tagged with 'make release'\n Commit: Updates date arithmetic to handle whitespace (#903)\n\n -- Jon Lee Thu, 09 Mar 2023 14:23:44 -0800\n\ntron (1.24.3) jammy; urgency=medium\n\n * 1.24.3 tagged with 'make release'\n Commit: Upgrades kubernetes version to support 1.21 (#902)\n\n -- Jon Lee Thu, 09 Mar 2023 12:15:55 -0800\n\ntron (1.24.2) jammy; urgency=medium\n\n * 1.24.2 tagged with 'make release'\n Commit: Ignores CryptographyDeprecationWarning (#901) * Ignores\n CryptographyDeprecationWarning\n\n -- Jon Lee Thu, 09 Mar 2023 11:46:44 -0800\n\ntron (1.24.1) jammy; urgency=medium\n\n * 1.24.1 tagged with 'make release'\n Commit: Upgrades requirements and adds caching on get loggers (#900)\n * updates axe-core * Upgrade twisted * Stop locking on getting\n loggers * Adds better description and commit suggestions *\n Upgrades Werkzeug dependency * Upgrades future dependency *\n Upgrades ipython dependency * Upgrades cryptography dependency *\n Upgrades setuptools dependency * Upgrades certifi dependency *\n Upgrades urllib3 dependency * Upgrades rsa dependency * Upgrades\n Pygments dependency * Upgrades jinja2 for docs dependency *\n Upgrades docs dependencies * Update tron/trondaemon.py Co-authored-\n by: Luis Pérez --------- Co-authored-by: Luis\n Pérez \n\n -- Jon Lee Wed, 08 Mar 2023 10:13:38 -0800\n\ntron (1.24.0) jammy; urgency=medium\n\n * 1.24.0 tagged with 'make release'\n Commit: Fixes make release for jammy (#899)\n\n -- Jon Lee Fri, 03 Mar 2023 05:39:51 -0800\n\ntron (1.23.10) bionic; urgency=medium\n\n * 1.23.10 tagged with 'make release'\n Commit: Merge pull request #890 from\n Yelp/u/cuza/logging_tron_exit_status Adding ExecStopPost to systemd\n unit file to log tron exit status\n\n -- root Tue, 31 Jan 2023 18:27:19 +0000\n\ntron (1.23.9) bionic; urgency=medium\n\n * 1.23.9 tagged with 'make release'\n Commit: Merge pull request #887 from Yelp/u/jfong/TRON-1723 TRON-\n 1723: Add user attribution into user agent when invoking tronctl\n commands\n\n -- root Wed, 07 Dec 2022 20:29:42 +0000\n\ntron (1.23.8) bionic; urgency=medium\n\n * 1.23.8 tagged with 'make release'\n Commit: TRON-1825: Add additional failure handling of specific\n errors (#886) Adds additional failure handling of specific errors\n Adds handling of spot interruptions and k8s scaling down Captures\n real exit codes from failures.\n\n -- root Mon, 05 Dec 2022 15:43:23 +0000\n\ntron (1.23.7) bionic; urgency=medium\n\n * 1.23.7 tagged with 'make release'\n Commit: Use correct dict casing for k8s metadata (#885) The data\n we'd been dumping into our logging streams was using different key\n names than what task_proc was actually sending k8s: we were logging\n the python-ified names, but task_proc was sending tron the literal\n k8s payloads (i.e., keynames being camelCased). We didn't notice\n this 'cause until the initial attempt at working around this weird\n k8s bug we weren't actually logging what tron was seeing and our\n test data in the unit tests was based on what we'd been seeing in\n our logging streams\n\n -- root Wed, 23 Nov 2022 19:44:24 +0000\n\ntron (1.23.6) bionic; urgency=medium\n\n * 1.23.6 tagged with 'make release'\n Commit: Detect abnormal successful exits in k8s (#884) this is\n kinda wild: we're seeing that a kubelet will sometimes fail to start\n a container (usually due to what appear to be race conditions like\n those mentioned in\n https://github.com/kubernetes/kubernetes/issues/100047#issuecomment-\n 797624208 and then decide that these Pods should be phase=Succeeded\n with an exit code of 0 - even though the container never actually\n started. So far, we've noticed that when this happens, the and\n fields will be - thus we'll check for at least one of these\n conditions to detect an abnormal exit and actually 'fail' the\n affected action\n\n -- root Tue, 22 Nov 2022 19:28:02 +0000\n\ntron (1.23.5) bionic; urgency=medium\n\n * 1.23.5 tagged with 'make release'\n Commit: Merge pull request #882 from Yelp/DAR-1739-revert-rookout\n Revert 'This adds rookout to tron (TRON-1764) (#875)'\n\n -- root Wed, 26 Oct 2022 00:24:33 +0000\n\ntron (1.23.4) bionic; urgency=medium\n\n * 1.23.4 tagged with 'make release'\n Commit: Bail out earlier on large tron logs (#881) It's possible\n for there to be multiple gigabytes of logs in our logging streams\n and we don't want Tron sitting there processing these for ages on\n end. This isn't a silver bullet, but this should help in the\n meantime\n\n -- root Mon, 24 Oct 2022 16:02:12 +0000\n\ntron (1.23.3) bionic; urgency=medium\n\n * 1.23.3 tagged with 'make release'\n Commit: TRON-1806 Adds new function in check_tron_jobs to skip\n failed logging queries for a specific superregion (#880) Adds new\n function in check_tron_jobs --skip-sensu-failure-logging\n\n -- root Fri, 07 Oct 2022 19:53:47 +0000\n\ntron (1.23.2) bionic; urgency=medium\n\n * 1.23.2 tagged with 'make release'\n Commit: fixes incorrect test\n\n -- root Thu, 06 Oct 2022 15:24:16 +0000\n\ntron (1.23.1) bionic; urgency=medium\n\n * 1.23.1 tagged with 'make release'\n Commit: Merge pull request #877 from Yelp/jammy-3.7 Python 3.7;\n build for Jammy\n\n -- root Thu, 15 Sep 2022 22:42:37 +0000\n\ntron (1.23.0) bionic; urgency=medium\n\n * 1.23.0 tagged with 'make release'\n Commit: This adds rookout to tron (TRON-1764) (#875) Adds rookout\n to tron (TRON-1764)\n\n -- root Mon, 12 Sep 2022 19:54:31 +0000\n\ntron (1.22.0) bionic; urgency=medium\n\n * 1.22.0 tagged with 'make release'\n Commit: Merge pull request #873 from Yelp/u/emanelsabban/tronUI-1737\n Limiting amount of output displayed in Tron UI - TRON-1737\n\n -- root Tue, 23 Aug 2022 19:35:57 +0000\n\ntron (1.21.0) bionic; urgency=medium\n\n * 1.21.0 tagged with 'make release'\n Commit: Merge pull request #874 from Yelp/u/jfong/TRON-1777-update-\n taskproc TRON-1777: bump taskproc to get kubeconfig reload fix\n\n -- root Mon, 15 Aug 2022 18:29:12 +0000\n\ntron (1.20.0) bionic; urgency=medium\n\n * 1.20.0 tagged with 'make release'\n Commit: bumped up taskproc ver pass tron version to tskprc\n\n -- root Mon, 25 Jul 2022 20:44:50 +0000\n\ntron (1.19.0) bionic; urgency=medium\n\n * 1.19.0 tagged with 'make release'\n Commit: Pass port/field selector envvars to task_proc (#869) we\n need this for spark drivers in k8s to work - and without the field\n selector env vars we're not able to fully adhere to the paasta\n workload contract\n\n -- root Mon, 18 Apr 2022 16:10:56 +0000\n\ntron (1.18.0) bionic; urgency=medium\n\n * 1.18.0 tagged with 'make release'\n Commit: Merge pull request #867 from Yelp/u/kawaiwan/retry-honors-\n triggers Add arg to for waiting for dependencies\n\n -- root Wed, 13 Apr 2022 18:41:37 +0000\n\ntron (1.17.1) bionic; urgency=medium\n\n * 1.17.1 tagged with 'make release'\n Commit: Create KubernetesActionRuns for executor: spark (#866) For\n now, we should be able to just use a KubernetesActionRun for Spark\n drivers - worst case, I think we'll just have a couple places where\n we branch on if we absolutely need to do something Spark-specific\n and can't just pass that down from paasta-tools This PR also\n removes spark_driver_service_account_name - I'm not sure what I was\n thinking initially - we don't really need anything to distinguish\n between the driver and executor SA since the executor SA is\n configured by the arguments used to start the driver (and thus we\n can just start the driver pod as we normally would by using the\n service account payload that we'd use for a normal tron-launched\n pod)\n\n -- root Mon, 21 Mar 2022 15:03:15 +0000\n\ntron (1.17.0) bionic; urgency=medium\n\n * 1.17.0 tagged with 'make release'\n Commit: Bump pysensu-yelp to pull in issuetype support (#864)\n https://github.com/Yelp/pysensu-yelp/pull/30 was released in pysensu-\n yelp==0.4.4 and allows users to set the issuetype for a ticket (so\n that sensu-created tickets aren't always of type (the default))\n\n -- root Thu, 10 Mar 2022 18:47:54 +0000\n\ntron (1.16.3) bionic; urgency=medium\n\n * 1.16.3 tagged with 'make release'\n Commit: Respect ActionRunAdapter max_lines param (#862) Callers can\n request a specific number of lines for logs - before this change we\n were ignoring that (except for the k8s metadata logs) and always\n returning all the data available.\n\n -- root Thu, 10 Feb 2022 22:45:16 +0000\n\ntron (1.16.3) bionic; urgency=medium\n\n * 1.16.3 tagged with 'make release'\n Commit: Respect ActionRunAdapter max_lines param (#862) Callers can\n request a specific number of lines for logs - before this change we\n were ignoring that (except for the k8s metadata logs) and always\n returning all the data available.\n\n -- root Thu, 10 Feb 2022 19:46:18 +0000\n\ntron (1.16.2) bionic; urgency=medium\n\n * 1.16.2 tagged with 'make release'\n Commit: Merge pull request #861 from Yelp/u/kawaiwan/always-restart-\n trond Always restart trond if it goes down\n\n -- root Thu, 03 Feb 2022 19:20:28 +0000\n\ntron (1.16.1) xenial; urgency=medium\n\n * 1.16.1 tagged with 'make release'\n Commit: Silence yelp_clog.StreamTailer logs + k8s event guard (#855)\n This should cleanup any non-fatal exceptions that we see in Tron\n logs.\n\n -- root Tue, 07 Dec 2021 19:39:33 +0000\n\ntron (1.16.0) xenial; urgency=medium\n\n * 1.16.0 tagged with 'make release'\n Commit: Allow users to use runid for triggers (#853) Some users\n have jobs whose output depends on the output of the previous run -\n for critical jobs, it's useful to explicitly declare this dependency\n so that these jobs never run unless Tron knows that the previous\n output is ready/complete (using successful completion as a proxy for\n this) For additional context:\n https://yelp.slack.com/archives/CA4K8PBLG/p1638402768180500\n\n -- root Mon, 06 Dec 2021 19:06:11 +0000\n\ntron (1.15.0) xenial; urgency=medium\n\n * 1.15.0 tagged with 'make release'\n Commit: Add Service Account support for k8s jobs (#852) We'll use\n this for Pod Identity - there's a webhook that will inject a secret\n token + some environment variables if a Pod has a Service Account\n set up. paasta-tools will be in charge of creating the Service\n Account that we'll use if it doesn't exist (as well as setting up\n the annotations on that Service Account).\n\n -- root Fri, 03 Dec 2021 20:41:48 +0000\n\ntron (1.14.5) xenial; urgency=medium\n\n * 1.14.5 tagged with 'make release'\n Commit: Allow passing Pod annotations to task_processing (#850)\n We'll need this to toggle certain behaviors (e.g., disable Pods\n getting routable IPs internally).\n\n -- root Thu, 28 Oct 2021 18:11:55 +0000\n\ntron (1.14.4) xenial; urgency=medium\n\n * 1.14.4 tagged with 'make release'\n Commit: Update scribereader to v0.14.1 (#847) v0.14.0 has a bug\n that points to the wrong hosts (corpdev uses dev infra, not its own)\n\n -- root Wed, 06 Oct 2021 19:44:17 +0000\n\ntron (1.14.3) xenial; urgency=medium\n\n * 1.14.3 tagged with 'make release'\n Commit: Upgrade scribereader so that we can read logs in corpdev\n (#846) We duplicate the ecosystems that we can point scribereader\n at in the package and that list was missing corpdev This causes a\n KeyError when trying to get the tailer host:port tuple for tron in\n corpdev\n\n -- root Fri, 01 Oct 2021 19:57:14 +0000\n\ntron (1.14.2) xenial; urgency=medium\n\n * 1.14.2 tagged with 'make release'\n Commit: Support passing Pod labels to task_processing (#841) We'll\n need this to set labels required by the PaaSTA Workload Contract\n (which is needed to ensure that internal tooling will work as\n expected with our Pods)\n\n -- root Fri, 10 Sep 2021 17:32:09 +0000\n\ntron (1.14.1) xenial; urgency=medium\n\n * 1.14.1 tagged with 'make release'\n Commit: Show duration in ActionRun history table (#840) This was\n missed when this view was initially created and is helpful for users\n to determine how run times for a given action are changing over time\n at a glance.\n\n -- root Thu, 02 Sep 2021 20:42:19 +0000\n\ntron (1.14.0) xenial; urgency=medium\n\n * 1.14.0 tagged with 'make release'\n Commit: Support node selectors/affinity for k8s tasks (#837) This\n additionally adds some error handling when things fail any\n invariants for a KubernetesTaskConfig - previously, an\n InvariantException meant that we would never realize that a\n KubernetesTaskConfig was invalid and that the ActionRun would never\n work.\n\n -- root Thu, 26 Aug 2021 16:00:51 +0000\n\ntron (1.13.2) xenial; urgency=medium\n\n * 1.13.2 tagged with 'make release'\n Commit: Use start and end times from ActionRunAttempts (#839) It\n turns out that we reset the start time for an ActionRun on any\n retries, so let's use the start time from the first attempt and the\n end time from the last attempt to figure out what timespan to ask\n for logs from scribereader\n\n -- root Tue, 24 Aug 2021 17:55:59 +0000\n\ntron (1.13.1) xenial; urgency=medium\n\n * 1.13.1 tagged with 'make release'\n Commit: Merge pull request #836 from Yelp/jfong/TRON-1658-\n kubernetesactionrunfromstate TRON-1658: Fix issue restoring\n KubernetesActionRuns\n\n -- root Wed, 18 Aug 2021 17:28:33 +0000\n\ntron (1.13.0) xenial; urgency=medium\n\n * 1.13.0 tagged with 'make release'\n Commit: Merge pull request #830 from Yelp/u/kawaiwan/automate-\n backfills Make tronctl backfill actually run backfills\n\n -- root Tue, 17 Aug 2021 16:38:44 +0000\n\ntron (1.12.0) xenial; urgency=medium\n\n * 1.12.0 tagged with 'make release'\n Commit: Actually kill k8s Pods on KubernetesActionRun::kill() (#828)\n We had left this method stubbed out until we implemented kill() in\n task_processing, but now we can actually finish implementation here.\n\n -- root Mon, 16 Aug 2021 22:03:29 +0000\n\ntron (1.11.0) xenial; urgency=medium\n\n * 1.11.0 tagged with 'make release'\n Commit: Support adding/dropping capabilities (#827) We'll need this\n for parity with the Mesos implementation and because it's a good\n security practice :p\n\n -- root Tue, 03 Aug 2021 19:32:46 +0000\n\ntron (1.10.3) xenial; urgency=medium\n\n * 1.10.3 tagged with 'make release'\n Commit: Merge pull request #824 from Yelp/u/kawaiwan/turn-on-\n taskproc-metrics Install yelp-meteorite when building yelp env\n\n -- root Mon, 02 Aug 2021 20:22:57 +0000\n\ntron (1.10.2) xenial; urgency=medium\n\n * 1.10.2 tagged with 'make release'\n Commit: Use un-typo'd failed platform_type for k8s events (#825)\n see https://github.com/Yelp/task_processing/pull/173 :p\n\n -- root Mon, 02 Aug 2021 18:56:46 +0000\n\ntron (1.10.1) xenial; urgency=medium\n\n * 1.10.1 tagged with 'make release'\n Commit: Merge pull request #822 from Yelp/jfong/TRON-1627-\n secret_env_to_taskproc TRON-1627: Pass secret_env to taskproc from\n kubernetes\n\n -- root Wed, 28 Jul 2021 22:43:21 +0000\n\ntron (1.10.0) xenial; urgency=medium\n\n * 1.10.0 tagged with 'make release'\n Commit: Enable submitting a task to k8s using task_proc (#818) *\n Enable submitting a task to k8s using task_proc We aren't yet\n reading any of the events that task_proc is bubbling back up to us,\n but that'll come next. With this, we should be giving task_proc all\n the information it needs to actually create a usable Pod.\n\n -- root Tue, 27 Jul 2021 20:24:40 +0000\n\ntron (1.9.1) xenial; urgency=medium\n\n * 1.9.1 tagged with 'make release'\n Commit: Release v1.9.0\n\n -- root Mon, 26 Jul 2021 19:30:29 +0000\n\ntron (1.8.1) xenial; urgency=medium\n\n * 1.8.1 tagged with 'make release'\n Commit: Add black for formatting + some additional linters (#805)\n We don't have any automatically enforced formatting (yapf is\n optional and not part of our pre-commit), so let's use black since\n that seems to be what we've settled on. Additionally, I've added\n some other 'standard' pre-commit hooks (as well as reformatted the\n pre-commit config file.)\n\n -- root Thu, 10 Jun 2021 18:52:48 +0000\n\ntron (1.8.0) xenial; urgency=medium\n\n * 1.8.0 tagged with 'make release'\n Commit: Add initial k8s config + global k8s toggle + job opt-out\n (#802) We'll be adding two toggles to control Tron's usage of k8s:\n * a global killswitch (k8s_options['enabled']) for when we want to\n go back to Mesos for an entire cluster (and, in the future, for\n when we want to quickly stop all Tronjobs). * a per-job opt-out ()\n for any jobs that encounter issues with k8s or that we want to\n migrate at a specific time. Since I was adding a k8s config section\n to the Tron master config, I also went ahead and added a way to\n configure what the k8s API address should be (i.e., the 'master'\n address).\n\n -- root Wed, 02 Jun 2021 16:01:28 +0000\n\ntron (1.7.0) xenial; urgency=medium\n\n * 1.7.0 tagged with 'make release'\n Commit: Implementation of skip-and-publish (#789) Skipping an\n action that has downstream triggers is error-prone as its easy to\n forget that you have triggers to emit and what triggers to emit. You\n probably need to go check your tronfig definitions to see what\n triggers exist and converting the trigger name to a trigger command\n can be a little tricky (with date magic). To solve this, we add a\n new tronctl command that will skip an action and then publish all\n triggers for that action.\n\n -- root Thu, 15 Apr 2021 17:15:50 +0000\n\ntron (1.6.1) xenial; urgency=medium\n\n * 1.6.1 tagged with 'make release'\n Commit: Merge pull request #780 from Yelp/TRON-1570-remove-old-state\n Stop saving command config and fields replaced by command config and\n …\n\n -- root Fri, 30 Oct 2020 16:39:41 +0000\n\ntron (1.6.0) xenial; urgency=medium\n\n * 1.6.0 tagged with 'make release'\n Commit: Merge pull request #778 from Yelp/TRON-1161-retry-configs\n TRON-1161: update configs for retries\n\n -- root Wed, 07 Oct 2020 20:23:20 +0000\n\ntron (1.5.1) xenial; urgency=medium\n\n * 1.5.1 tagged with 'make release'\n Commit: Merge pull request #777 from Yelp/only-reconfigure-namespace\n Only reconfigure jobs in that namespace when a namespace is updated\n\n -- root Thu, 01 Oct 2020 18:18:18 +0000\n\ntron (1.5.0) xenial; urgency=medium\n\n * 1.5.0 tagged with 'make release'\n Commit: Merge pull request #775 from Yelp/TRON-1563-retries-separate\n TRON-1563: save state and configs for retries independently\n\n -- root Tue, 15 Sep 2020 00:33:05 +0000\n\ntron (1.4.6) xenial; urgency=medium\n\n * 1.4.6 tagged with 'make release'\n Commit: Merge pull request #774 from Yelp/drmorr/TRON-\n 1566/redact_aws_keys redact AWS credentials from the logs\n\n -- root Wed, 09 Sep 2020 17:44:42 +0000\n\ntron (1.4.5) xenial; urgency=medium\n\n * 1.4.5 tagged with 'make release'\n Commit: Merge pull request #773 from Yelp/TRON-1564-allow-check-oom-\n events Skip check_oom_events key from monitoring, which is only\n used by paasta\n\n -- root Thu, 03 Sep 2020 17:30:12 +0000\n\ntron (1.4.4) xenial; urgency=medium\n\n * 1.4.4 tagged with 'make release'\n Commit: Merge pull request #769 from Yelp/try-react Prototype React\n version of tronweb\n\n -- root Fri, 28 Aug 2020 18:20:16 +0000\n\ntron (1.4.3) xenial; urgency=medium\n\n * 1.4.3 tagged with 'make release'\n Commit: Merge pull request #770 from Yelp/TRON-1561-no-overlap-alert-\n if-queueing Do not alert for overlapping runs if queueing is\n disabled\n\n -- root Wed, 26 Aug 2020 21:02:42 +0000\n\ntron (1.4.2) xenial; urgency=medium\n\n * 1.4.2 tagged with 'make release'\n Commit: Merge pull request #765 from Yelp/drmorr/TRON-\n 1554/fix_pypi_url don't use public pypi for internal builds\n\n -- root Tue, 04 Aug 2020 18:23:42 +0000\n\ntron (1.4.1) xenial; urgency=medium\n\n * 1.4.1 tagged with 'make release'\n Commit: v1.4.1\n\n -- root Thu, 30 Jul 2020 23:56:44 +0000\n\ntron (1.4.0) xenial; urgency=medium\n\n * 1.4.0 tagged with 'make release'\n Commit: Merge branch 'TRON-1527-separate-job-run-state'\n\n -- root Tue, 28 Jul 2020 22:24:08 +0000\n\ntron (1.3.15) xenial; urgency=medium\n\n * 1.3.15 tagged with 'make release'\n Commit: Merge pull request #757 from Yelp/dedup-save-queue De-\n duplicate items in the DynamoDB save queue\n\n -- root Thu, 23 Jul 2020 20:01:54 +0000\n\ntron (1.3.14) xenial; urgency=medium\n\n * 1.3.14 tagged with 'make release'\n Commit: Merge pull request #756 from Yelp/TRON-1539-dynamo-metrics\n Tron 1539 dynamo metrics\n\n -- root Wed, 01 Jul 2020 21:26:52 +0000\n\ntron (1.3.13) xenial; urgency=medium\n\n * 1.3.13 tagged with 'make release'\n Commit: Merge pull request #755 from Yelp/drmorr/COMPINFRA-\n 333/bump_requirements Bump requirements to pick up new task_proc\n version\n\n -- root Wed, 06 May 2020 22:55:55 +0000\n\ntron (1.3.12) xenial; urgency=medium\n\n * 1.3.12 tagged with 'make release'\n Commit: Merge pull request #754 from Yelp/u/dpopes/TRON-1531-\n keyboard-interactive-suppress-tty-if-no-prompt TRON-1531: Suppress\n the need to prompt the user via tty\n\n -- root Tue, 05 May 2020 18:26:53 +0000\n\ntron (1.3.11) xenial; urgency=medium\n\n * 1.3.11 tagged with 'make release'\n Commit: Merge pull request #753 from Yelp/u/dpopes/SEC-12778-support-\n keyboard-interactive-ssh Support keyboard-interactive as an\n authentication method for ssh\n\n -- root Mon, 27 Apr 2020 18:01:41 +0000\n\ntron (1.3.10) xenial; urgency=medium\n\n * 1.3.10 tagged with 'make release'\n Commit: block new saves if save queue is too big\n\n -- root Tue, 14 Apr 2020 16:51:46 +0000\n\ntron (1.3.9) xenial; urgency=medium\n\n * 1.3.9 tagged with 'make release'\n Commit: fix tests, fix issue where stopping condition can't be\n reached\n\n -- root Tue, 14 Apr 2020 14:46:43 +0000\n\ntron (1.3.8) xenial; urgency=medium\n\n * 1.3.8 tagged with 'make release'\n Commit: consume save queue in predefined chunks, count errors\n correctly\n\n -- root Tue, 14 Apr 2020 14:07:12 +0000\n\ntron (1.3.7) xenial; urgency=medium\n\n * 1.3.7 tagged with 'make release'\n Commit: Merge pull request #749 from Yelp/u/maksym/fix-redirect Fix\n redirect, comment out stuff in tronrepl\n\n -- root Wed, 08 Apr 2020 23:17:14 +0000\n\ntron (1.3.6) xenial; urgency=medium\n\n * 1.3.6 tagged with 'make release'\n Commit: Merge pull request #745 from Yelp/dependabot/pip/psutil-\n 5.6.6 Bump psutil from 5.6.3 to 5.6.6\n\n -- root Thu, 02 Apr 2020 23:56:54 +0000\n\ntron (1.3.5) xenial; urgency=medium\n\n * 1.3.5 tagged with 'make release'\n Commit: Merge pull request #744 from Yelp/mbehrens-TRON-1144-add-\n uptime-version Add tron version and uptime to frontend navbar\n\n -- root Mon, 16 Mar 2020 17:38:19 +0000\n\ntron (1.3.4) xenial; urgency=medium\n\n * 1.3.4 tagged with 'make release'\n Commit: Merge pull request #741 from Yelp/fix-tz-forward Fix bug\n from fall forward last year\n\n -- root Fri, 06 Mar 2020 18:59:59 +0000\n\ntron (1.3.3) xenial; urgency=medium\n\n * 1.3.3 tagged with 'make release'\n Commit: Merge pull request #737 from Yelp/u/siruitan/TRON-385-\n skip_validate_in_write_config Skip job graph validation in\n write_config\n\n -- root Thu, 06 Feb 2020 18:59:31 +0000\n\ntron (1.3.2) xenial; urgency=medium\n\n * 1.3.2 tagged with 'make release'\n Commit: Merge branch 'drmorr/TRON-\n 1369/fixing_the_clusterman_invocation'\n\n -- root Wed, 05 Feb 2020 23:32:42 +0000\n\ntron (1.3.1) xenial; urgency=medium\n\n * 1.3.1 tagged with 'make release'\n Commit: Merge pull request #735 from Yelp/drmorr/TRON-\n 1454/shorten_tron_directory_names tron output dir names shortened\n\n -- root Tue, 04 Feb 2020 18:35:32 +0000\n\ntron (1.3.0) xenial; urgency=medium\n\n * 1.3.0 tagged with 'make release'\n Commit: Merge branch 'drmorr/TRON-1369/clusterman_env_var'\n\n -- root Thu, 16 Jan 2020 18:59:08 +0000\n\ntron (1.2.5) xenial; urgency=medium\n\n * 1.2.5 tagged with 'make release'\n Commit: Merge branch 'u/kawaiwan/bump-task-proc-to-0.1.8'\n\n -- root Tue, 12 Nov 2019 01:01:46 +0000\n\ntron (1.2.4) xenial; urgency=medium\n\n * 1.2.4 tagged with 'make release'\n Commit: Merge pull request #723 from Yelp/drmorr/TRON-\n 1329/fix_time_interval_construction fix time interval construction\n in check_tron_jobs\n\n -- root Wed, 06 Nov 2019 21:56:14 +0000\n\ntron (1.2.3) xenial; urgency=medium\n\n * 1.2.3 tagged with 'make release'\n Commit: Merge pull request #717 from Yelp/monitoring-backfill-\n buckets Try to handle backfills better in monitoring precious runs\n\n -- root Tue, 15 Oct 2019 23:27:13 +0000\n\ntron (1.2.2) xenial; urgency=medium\n\n * 1.2.2 tagged with 'make release'\n Commit: Merge branch 'drmorr/TRON-\n 1303/action_runner_sets_env_variables'\n\n -- root Mon, 14 Oct 2019 19:47:54 +0000\n\ntron (1.2.1) xenial; urgency=medium\n\n * 1.2.1 tagged with 'make release'\n Commit: Merge pull request #716 from Yelp/retry-check-state Do not\n auto retry action if state is already succeeded\n\n -- root Wed, 09 Oct 2019 16:57:32 +0000\n\ntron (1.2.0) xenial; urgency=medium\n\n * 1.2.0 tagged with 'make release'\n Commit: Merge pull request #715 from Yelp/auto-recovery\n Automatically try to recover on unknown ssh actions\n\n -- root Wed, 09 Oct 2019 00:17:59 +0000\n\ntron (1.1.0) xenial; urgency=medium\n\n * 1.1.0 tagged with 'make release'\n Commit: Merge pull request #714 from Yelp/recover-command Add\n manual recovery command to api\n\n -- root Thu, 03 Oct 2019 16:54:12 +0000\n\ntron (1.0.4) xenial; urgency=medium\n\n * 1.0.4 tagged with 'make release'\n Commit: Merge pull request #708 from Yelp/recovery-check-finished\n Put back check for action runners that have completed in recovery\n\n -- root Tue, 24 Sep 2019 00:50:16 +0000\n\ntron (1.0.3) xenial; urgency=medium\n\n * 1.0.3 tagged with 'make release'\n Commit: Merge pull request #706 from\n Yelp/fix_check_tron_jobs_job_run_ids fix job run ids reported by\n check_tron_jobs\n\n -- root Thu, 19 Sep 2019 21:56:23 +0000\n\ntron (1.0.2) xenial; urgency=medium\n\n * 1.0.2 tagged with 'make release'\n Commit: Don't do a retry_delay when manually retrying a failed\n command. (#704)\n\n -- root Wed, 18 Sep 2019 20:28:54 +0000\n\ntron (1.0.1) xenial; urgency=medium\n\n * 1.0.1 tagged with 'make release'\n Commit: Merge pull request #692 from Yelp/taskproc_bump Bump\n taskproc to handle weird staging offers\n\n -- root Wed, 07 Aug 2019 13:37:07 +0000\n\ntron (1.0.0) xenial; urgency=medium\n\n * 1.0.0 tagged with 'make release'\n Commit: Released 0.9.14.6 via make release\n\n -- root Fri, 19 Jul 2019 23:23:34 +0000\n\ntron (0.9.14.6) xenial; urgency=medium\n\n * 0.9.14.6 tagged with 'make release'\n Commit: Merge pull request #688 from Yelp/sort_order_fixes Make\n string sorting only in ActionRun displaying.\n\n -- root Tue, 16 Jul 2019 21:46:48 +0000\n\ntron (0.9.14.5) xenial; urgency=medium\n\n * 0.9.14.5 tagged with 'make release'\n Commit: Released 0.9.14.4 via make release\n\n -- root Tue, 16 Jul 2019 18:23:28 +0000\n\ntron (0.9.14.4) xenial; urgency=medium\n\n * 0.9.14.4 tagged with 'make release'\n Commit: Merge pull request #687 from Yelp/fix_tron_sorting Fix\n sorting for DisplayActionRuns when fields are None\n\n -- root Tue, 16 Jul 2019 18:21:47 +0000\n\ntron (0.9.14.4) xenial; urgency=medium\n\n * 0.9.14.4 tagged with 'make release'\n Commit: Merge pull request #682 from Yelp/stderr_fix Fix stderr\n output on check_tron_jobs\n\n -- root Fri, 12 Jul 2019 16:56:55 +0000\n\ntron (0.9.14.3) xenial; urgency=medium\n\n * 0.9.14.3 tagged with 'make release'\n Commit: Merge pull request #681 from Yelp/refix-scheduling Use last\n run time to get next run time before scheduling\n\n -- root Thu, 11 Jul 2019 18:11:28 +0000\n\ntron (0.9.14.2) xenial; urgency=medium\n\n * 0.9.14.2 tagged with 'make release'\n Commit: Merge pull request #678 from Yelp/ssh-timeouts Improve\n logging for deferred errors and apply timeout config to trans…\n\n -- root Fri, 05 Jul 2019 20:32:44 +0000\n\ntron (0.9.14.1) xenial; urgency=medium\n\n * 0.9.14.1 tagged with 'make release'\n Commit: Merge branch 'master' of github.com:Yelp/Tron\n\n -- root Mon, 03 Jun 2019 19:01:33 +0000\n\ntron (0.9.14.0) xenial; urgency=medium\n\n * 0.9.14.0 tagged with 'make release'\n Commit: Merge pull request #668 from Yelp/another-waiting-fix Runs\n with triggers should start in scheduled state\n\n -- root Thu, 16 May 2019 17:30:06 +0000\n\ntron (0.9.13.4) xenial; urgency=medium\n\n * 0.9.13.4 tagged with 'make release'\n Commit: Merge pull request #667 from Yelp/revert-665-revert-654-\n u/mingqiz/disable_shelvestore removed mirror store\n\n -- root Tue, 14 May 2019 23:09:44 +0000\n\ntron (0.9.13.3) xenial; urgency=medium\n\n * 0.9.13.3 tagged with 'make release'\n Commit: Merge pull request #666 from Yelp/u/kawaiwan/improve-\n recovery Make recovery batch script also check if action runner\n suddenly goes away\n\n -- root Tue, 14 May 2019 00:08:31 +0000\n\ntron (0.9.13.2) xenial; urgency=medium\n\n * 0.9.13.2 tagged with 'make release'\n Commit: fix guess_realert when next_run is the same as previous_run\n (#658) fix guess_realert when next_run is the same as previous_run\n\n -- root Thu, 02 May 2019 17:48:01 +0000\n\ntron (0.9.13.1) xenial; urgency=medium\n\n * 0.9.13.1 tagged with 'make release'\n Commit: Merge pull request #656 from Yelp/u/mingqiz/fix_dynamodb_bug\n Changed dynamodb partition index to int so it is sorted correctly\n\n -- root Tue, 30 Apr 2019 20:05:19 +0000\n\ntron (0.9.13.0) xenial; urgency=medium\n\n * 0.9.13.0 tagged with 'make release'\n Commit: Merge pull request #649 from Yelp/drmorr/TRON-\n 1095/add_duration_to_tronweb duration field added to tronweb\n\n -- root Mon, 22 Apr 2019 23:16:52 +0000\n\ntron (0.9.12.6) xenial; urgency=medium\n\n * 0.9.12.6 tagged with 'make release'\n Commit: Merge pull request #648 from Yelp/u/mingqiz/TRON-1088-bin\n Removed interval scheduler\n\n -- root Fri, 19 Apr 2019 18:11:18 +0000\n\ntron (0.9.12.5) xenial; urgency=medium\n\n * 0.9.12.5 tagged with 'make release'\n Commit: Merge branch 'drmorr/TRON-605/fix_monitoring_for_stuck_jobs'\n\n -- root Mon, 15 Apr 2019 21:02:28 +0000\n\ntron (0.9.12.4) xenial; urgency=medium\n\n * 0.9.12.4 tagged with 'make release'\n Commit: Merge pull request #640 from Yelp/u/mingqiz/TRON-638-\n improve_read_speed Improve dynamodb read/write speed\n\n -- root Wed, 10 Apr 2019 22:59:47 +0000\n\ntron (0.9.12.3) xenial; urgency=medium\n\n * 0.9.12.3 tagged with 'make release'\n Commit: Merge pull request #639 from Yelp/u/mingqiz/TRON-1041-\n open_file_limit changed max number of open files to 10000\n\n -- root Wed, 03 Apr 2019 00:39:28 +0000\n\ntron (0.9.12.2) xenial; urgency=medium\n\n * 0.9.12.2 tagged with 'make release'\n Commit: tron/__init__.py\n\n -- root Tue, 02 Apr 2019 18:10:28 +0000\n\ntron (0.9.12.1) xenial; urgency=medium\n\n * 0.9.12.1 tagged with 'make release'\n Commit: Merge pull request #631 from Yelp/u/mingqiz/TRON-638-\n fix_validation fixed data validation of dynamodb migration\n\n -- root Fri, 22 Mar 2019 18:06:11 +0000\n\ntron (0.9.12.0) xenial; urgency=medium\n\n * 0.9.12.0 tagged with 'make release'\n Commit: Merge branch 'drmorr/TRON-390/xjob_dep_viz'\n\n -- root Wed, 13 Mar 2019 18:39:40 +0000\n\ntron (0.9.11.1) xenial; urgency=medium\n\n * 0.9.11.1 tagged with 'make release'\n Commit: Merge pull request #628 from Yelp/u/mingqiz/TRON-661 Fixed\n trusty build\n\n -- root Tue, 12 Mar 2019 00:32:40 +0000\n\ntron (0.9.11.0) xenial; urgency=medium\n\n * 0.9.11.0 tagged with 'make release'\n Commit: Merge pull request #617 from Yelp/u/mingqiz/TRON-638-\n migration Migrating from Berkley DB to DynamoDB (TRON-638)\n\n -- root Fri, 08 Mar 2019 17:27:53 +0000\n\ntron (0.9.10.0) xenial; urgency=medium\n\n * 0.9.10.0 tagged with 'make release'\n Commit: Merge pull request #622 from Yelp/waiting-state Waiting\n state if an action is waiting for normal or cross-job dependencies\n\n -- root Mon, 04 Mar 2019 19:48:20 +0000\n\ntron (0.9.9.15) xenial; urgency=medium\n\n * 0.9.9.15 tagged with 'make release'\n Commit: Merge pull request #624 from Yelp/add-waiting-state Add\n waiting state first for rollback safety\n\n -- root Mon, 04 Mar 2019 17:53:07 +0000\n\ntron (0.9.9.14) trusty; urgency=medium\n\n * 0.9.9.14 tagged with 'make release'\n Commit: Merge pull request #615 from Yelp/fi'-check-job-duration\n Ignore duration=None for jobs waiting on e'ternal dependency\n\n -- root Fri, 08 Feb 2019 12:31:54 +0000\n\ntron (0.9.9.13) trusty; urgency=medium\n\n * 0.9.9.13 tagged with 'make release'\n Commit: fi' disk=none error when restoring from state\n\n -- root Thu, 07 Feb 2019 16:18:18 +0000\n\ntron (0.9.9.12) trusty; urgency=medium\n\n * 0.9.9.12 tagged with 'make release'\n Commit: Merge pull request #613 from Yelp/fix-jobrun-state-proxy\n Fix inconsistency between JobRun state attribute and is_\n checks\n\n -- root Thu, 07 Feb 2019 15:14:29 +0000\n\ntron (0.9.9.11) trusty; urgency=medium\n\n * 0.9.9.11 tagged with 'make release'\n Commit: Merge pull request #610 from Yelp/disk_support Added disk\n support to tron on mesos\n\n -- root Mon, 04 Feb 2019 18:50:33 +0000\n\ntron (0.9.9.10) trusty; urgency=medium\n\n * 0.9.9.10 tagged with 'make release'\n Commit: Merge pull request #611 from Yelp/fix-job-appearing-pending\n Fix job appearing pending when waiting on trigger requirement\n\n -- root Thu, 31 Jan 2019 11:44:45 +0000\n\ntron (0.9.9.9) trusty; urgency=medium\n\n * 0.9.9.9 tagged with 'make release'\n Commit: tron.config.schema: use enum types values e'plicitly to fi'\n regression after migrating to native enums\n\n -- root Mon, 07 Jan 2019 13:46:43 +0000\n\ntron (0.9.9.8) trusty; urgency=medium\n\n * 0.9.9.8 tagged with 'make release'\n Commit: Merge pull request #603 from Yelp/recovery-none-action-runs\n Skip job runs with no action runs during recovery\n\n -- root Thu, 20 Dec 2018 19:19:02 +0000\n\ntron (0.9.9.7) trusty; urgency=medium\n\n * 0.9.9.7 tagged with 'make release'\n * Fix type bug in creating tasks during recovery\n * Only recover Mesos actions with no end time\n\n -- root Tue, 18 Dec 2018 19:27:22 +0000\n\ntron (0.9.9.6) trusty; urgency=medium\n\n * 0.9.9.6 tagged with 'make release'\n * Fixes for inactive framework\n\n -- root Thu, 06 Dec 2018 00:59:04 +0000\n\ntron (0.9.9.5) trusty; urgency=medium\n\n * 0.9.9.5 tagged with 'make release'\n * Allow passing extra options to systemd-based distros.\n * Update end times for actions\n * Fix bug related to Mesos offer timeouts\n * Do not reschedule disabled job if it is reconfigured\n\n -- root Tue, 04 Dec 2018 21:38:20 +0000\n\ntron (0.9.9.4) trusty; urgency=medium\n\n * 0.9.9.4 tagged with 'make release'\n - Fix signal handling so that Tron gracefull shuts down\n - Add more information to log lines\n\n -- root Mon, 19 Nov 2018 23:59:43 +0000\n\ntron (0.9.9.3) trusty; urgency=medium\n\n * 0.9.9.3 tagged with 'make release'\n Commit: Merge pull request #587 from Yelp/TRON-536 Removed spaces\n in action run html\n\n -- root Thu, 15 Nov 2018 00:20:45 +0000\n\ntron (0.9.9.2) trusty; urgency=medium\n\n * 0.9.9.2 tagged with 'make release'\n Commit: Merge pull request #583 from Yelp/u/kawaiwan/add-cluster-to-\n tron-metrics Add cluster option to metrics script\n\n -- root Mon, 12 Nov 2018 18:25:34 +0000\n\ntron (0.9.9.1) trusty; urgency=medium\n\n * 0.9.9.1 tagged with 'make release'\n Commit: modify action_id during job migration (#580) * modify\n action_id during job migration * pin pre-commit 1.11.2\n\n -- root Tue, 30 Oct 2018 21:42:21 +0000\n\ntron (0.9.9.0) trusty; urgency=medium\n\n * 0.9.9.0 tagged with 'make release'\n * Tronctl publish/discard events\n * Update DST fall back behavior to be consistent\n * Update requirements\n * Add script for fetching metrics\n\n -- root Tue, 30 Oct 2018 17:54:04 +0000\n\ntron (0.9.8.4) trusty; urgency=medium\n\n * 0.9.8.4 tagged with 'make release'\n Commit: Revert 'Merge pull request #570 from Yelp/pin-requirements'\n This reverts commit 33fb19df2d2ec508d254f209c1515bbffabc1523,\n reversing changes made to b3c7ab8ae8ffa2f7a08e275175989ea5d8d54a8a.\n\n -- root Fri, 26 Oct 2018 21:36:49 +0000\n\ntron (0.9.8.3) trusty; urgency=medium\n\n * 0.9.8.3 tagged with 'make release'\n Commit: Merge pull request #576 from Yelp/yapf-it yapf -rip\n\n -- root Fri, 26 Oct 2018 17:16:09 +0000\n\ntron (0.9.8.2) trusty; urgency=medium\n\n * 0.9.8.2 tagged with 'make release'\n Commit: Merge pull request #576 from Yelp/yapf-it yapf -rip\n\n -- root Fri, 26 Oct 2018 17:14:02 +0000\n\ntron (0.9.8.2) trusty; urgency=medium\n\n * 0.9.8.2 tagged with 'make release'\n Commit: Merge pull request #574 from\n Yelp/u/chl/rename_all_job_name_after_migration rename all job_name\n after migration\n\n -- root Thu, 25 Oct 2018 22:38:54 +0000\n\ntron (0.9.8.1) trusty; urgency=medium\n\n * 0.9.8.1 tagged with 'make release'\n Commit: Merge pull request #569 from Yelp/signal-none Don't log\n other signals\n\n -- root Tue, 23 Oct 2018 23:30:47 +0000\n\ntron (0.9.8.0) trusty; urgency=medium\n\n * 0.9.8.0 tagged with 'make release'\n Commit: Merge pull request #566 from Yelp/monitoring-allow-overlap\n If runs are allowed to overlap, don't consider that case stuck\n\n -- root Tue, 23 Oct 2018 20:51:02 +0000\n\ntron (0.9.7.0) trusty; urgency=medium\n\n * 0.9.7.0 tagged with 'make release'\n Commit: fix typo in systemd unit file\n\n -- root Wed, 17 Oct 2018 17:37:58 +0000\n\ntron (0.9.6.5) trusty; urgency=medium\n\n * 0.9.6.5 tagged with 'make release'\n Commit: Merge pull request #561 from Yelp/fix-dependent Fix\n actiongraph adapter failing to render dependent actions\n\n -- root Tue, 16 Oct 2018 17:37:45 +0000\n\ntron (0.9.6.4) trusty; urgency=medium\n\n * 0.9.6.4 tagged with 'make release'\n Commit: Released 0.9.6.3 via make release\n\n -- root Mon, 15 Oct 2018 18:27:07 +0000\n\ntron (0.9.6.3) trusty; urgency=medium\n\n * 0.9.6.3 tagged with 'make release'\n Commit: Merge pull request #555 from Yelp/u/chl/TRON-\n 442_create_migrate_job_tool add tronctl move command\n\n -- root Mon, 15 Oct 2018 17:52:20 +0000\n\ntron (0.9.6.2) trusty; urgency=medium\n\n * 0.9.6.2 tagged with 'make release'\n Commit: Merge pull request #548 from Yelp/skip-docs-diagram Skip\n state diagram in docs\n\n -- root Tue, 02 Oct 2018 12:23:05 +0000\n\ntron (0.9.6.1) trusty; urgency=medium\n\n * 0.9.6.1 tagged with 'make release'\n Commit: fix format string bugs (#538) * fix format string bugs\n\n -- root Fri, 21 Sep 2018 23:46:13 +0000\n\ntron (0.9.6.0) trusty; urgency=medium\n\n * 0.9.6.0 tagged with 'make release'\n * Remove percent string support\n * Fix bug with fail and retries\n * Remove headers from tronfig\n * Emit triggers for cross-job dependencies\n\n -- root Thu, 20 Sep 2018 01:20:15 +0000\n\ntron (0.9.5.1) trusty; urgency=medium\n\n * 0.9.5.1 tagged with 'make release'\n * Update taskproc to 0.1.2 for Mesos fixes\n * Save jobs when re-configured\n * Fix tronweb CSS\n * Make scheme optional for Mesos master address\n * More feedback on killing Mesos actions\n\n -- root Mon, 10 Sep 2018 21:14:06 +0000\n\ntron (0.9.5.0) trusty; urgency=medium\n\n * 0.9.5.0 tagged with 'make release'\n * Increase upstart timeout\n * Remove enableall/disableall from jobs controller\n * Deprecate --nodaemon\n\n -- root Wed, 05 Sep 2018 22:04:23 +0000\n\ntron (0.9.4.0) trusty; urgency=medium\n\n * 0.9.4.0 tagged with 'make release'\n * add string format support (#490)\n * recover Mesos action runs on restart\n * run reactor on separate thread\n * job trigger configs and eventbus\n\n -- root Tue, 04 Sep 2018 22:23:25 +0000\n\ntron (0.9.3.0) trusty; urgency=medium\n\n * 0.9.3.0 tagged with 'make release'\n Commit: make master address optional (#513)\n\n -- root Fri, 24 Aug 2018 22:42:36 +0000\n\ntron (0.9.2.1) trusty; urgency=medium\n\n * 0.9.2.1 tagged with 'make release'\n Commit: Merge pull request #508 from Yelp/nix_testify Nix testify\n\n -- root Wed, 22 Aug 2018 10:53:38 +0000\n\ntron (0.9.2.0) trusty; urgency=medium\n\n * 0.9.2.0 tagged with 'make release'\n Commit: catch command rendering type error (#488) * catch command\n rendering type error\n\n -- root Thu, 09 Aug 2018 20:46:45 +0000\n\ntron (0.9.1.9) trusty; urgency=medium\n\n * 0.9.1.9 tagged with 'make release'\n Commit: Merge pull request #483 from Yelp/retries-delay-kill\n Retries delay: kill delayed action correctly\n\n -- root Tue, 24 Jul 2018 11:00:12 +0000\n\ntron (0.9.1.8) trusty; urgency=medium\n\n * 0.9.1.8 tagged with 'make release'\n * fix _get_seconds_from_duration bug in monitoring\n\n -- root Tue, 10 Jul 2018 20:49:53 +0000\n\ntron (0.9.1.7) trusty; urgency=medium\n\n * 0.9.1.7 tagged with 'make release'\n * Make unknown alerts critical instead of warning\n * Add manhole for debugging\n * Fix validation of full tronfig directory\n\n -- root Mon, 09 Jul 2018 17:35:38 +0000\n\ntron (0.9.1.6) trusty; urgency=medium\n\n * 0.9.1.6 tagged with 'make release'\n * Bug fixes\n * Remove graceful shutdown\n * Mesos: Add default volume configs, implement kill/stop commmands\n\n -- root Tue, 03 Jul 2018 17:08:51 +0000\n\ntron (0.9.1.5) trusty; urgency=medium\n\n * 0.9.1.5 tagged with 'make release'\n Commit: Merge pull request #470 from Yelp/fix-output-dir Check\n output dir first\n\n -- root Mon, 25 Jun 2018 19:27:00 +0000\n\ntron (0.9.1.4) trusty; urgency=medium\n\n * 0.9.1.4 tagged with 'make release'\n Commit: Merge pull request #469 from Yelp/u/robj/improve-action-\n runner action_runner logs to the output_dir; add timestamps to logs\n\n -- root Mon, 25 Jun 2018 17:00:57 +0000\n\ntron (0.9.1.3) trusty; urgency=medium\n\n * 0.9.1.3 tagged with 'make release'\n Commit: Merge pull request #468 from Yelp/u/robj/keep-fs-reasders-\n alive handle failures streaming to stdout/stderr\n\n -- root Fri, 22 Jun 2018 17:21:54 +0000\n\ntron (0.9.1.2) trusty; urgency=medium\n\n * 0.9.1.2 tagged with 'make release'\n Commit: Merge pull request #462 from Yelp/missing-deps add missing\n requests and psutil deps\n\n -- root Wed, 20 Jun 2018 17:56:19 +0000\n\ntron (0.9.1.1) trusty; urgency=medium\n\n * 0.9.1.1 tagged with 'make release'\n Commit: bump version\n\n -- root Fri, 15 Jun 2018 09:34:03 +0000\n\ntron (0.9.1.0) trusty; urgency=medium\n\n * 0.9.1.0 tagged with 'make release'\n Commit: Merge pull request #452 from Yelp/mesos-logging Get output\n from Mesos tasks\n\n -- root Wed, 13 Jun 2018 13:48:43 +0000\n\ntron (0.9.0.0) trusty; urgency=medium\n\n * 0.9.0.0 tagged with 'make release'\n Commit: Merge pull request #451 from Yelp/fix-machine-state-during-\n recovery set the machine state to running before recovery\n\n -- root Tue, 05 Jun 2018 16:48:52 +0000\n\ntron (0.8.0.6) trusty; urgency=medium\n\n * 0.8.0.6 tagged with 'make release'\n * Support for expected runtime alerts\n * Pre calculate state machine transitions\n * Bug fixes\n\n -- root Wed, 16 May 2018 18:31:23 +0000\n\ntron (0.8.0.5) trusty; urgency=medium\n\n * 0.8.0.5 tagged with 'make release'\n Commit: Merge pull request #435 from Yelp/encode-stdout\n maybe_encode all data in the file serializer\n\n -- root Tue, 24 Apr 2018 02:46:08 +0000\n\ntron (0.8.0.4) trusty; urgency=medium\n\n * 0.8.0.4 tagged with 'make release'\n Commit: friendlier output from tronctl retry\n\n -- root Fri, 20 Apr 2018 14:15:05 +0000\n\ntron (0.8.0.3) trusty; urgency=medium\n\n * 0.8.0.3 tagged with 'make release'\n Feature: Job actions can now be re-tried using cli command\n `tronctl retry `. This will automatically\n trigger dependent actions upon success of re-tried action.\n\n -- root Thu, 19 Apr 2018 13:14:04 +0000\n\ntron (0.8.0.2) trusty; urgency=medium\n\n * 0.8.0.2 tagged with 'make release'\n Fix: regression in tronweb jobs list\n\n -- root Wed, 18 Apr 2018 11:09:19 +0000\n\ntron (0.8.0.1) trusty; urgency=medium\n\n * 0.8.0.1 tagged with 'make release'\n Commit: Merge pull request #370 from Yelp/python3-deb Python3 deb\n\n -- root Mon, 16 Apr 2018 14:38:29 +0000\n\ntron (0.8.0.0) trusty; urgency=medium\n\n * 0.8.0.0 tagged with 'make release'\n Commit: remove duplicate ignore for debian/debhelper-build-stamp\n\n -- root Wed, 14 Mar 2018 11:14:08 +0000\n\ntron (0.7.8.3) trusty; urgency=medium\n\n * 0.7.8.3 tagged with 'make release'\n Commit: Merge pull request #426 from Yelp/maybe-decode-all-the-\n things Maybe decode all the things\n\n -- root Thu, 12 Apr 2018 13:29:50 +0000\n\ntron (0.7.8.2) trusty; urgency=medium\n\n * 0.7.8.2 tagged with 'make release'\n Commit: Merge pull request #424 from Yelp/cleanup-retries-validation\n Fix validation of cleanup action\n\n -- root Tue, 10 Apr 2018 11:52:29 +0000\n\ntron (0.7.8.1) trusty; urgency=medium\n\n * 0.7.8.1 tagged with 'make release'\n * Retries attribute for action runs\n * Preparing for Python 3 upgrade\n\n -- root Fri, 06 Apr 2018 14:45:12 +0000\n\ntron (0.7.8.0) trusty; urgency=medium\n\n * 0.7.8.0 tagged with 'make release'\n * Script to clean up namespaces\n * Improve check_tron_jobs logging\n * Fix bug in date context math with timezones\n * Improve tab completion\n * Remove more service code\n * Use config values to create PaaSTA action run that prints\n\n -- root Tue, 03 Apr 2018 18:51:45 +0000\n\ntron (0.7.7.1) trusty; urgency=medium\n\n * 0.7.7.1 tagged with 'make release'\n Commit: Preparing 0.7.7.1 release - fix shelve regression - use\n bsddb3 directly - remove service functionality from tronweb - add\n CORS header - dockerize itests - test debian package in travis - tab\n completion improvements\n\n -- root Fri, 23 Mar 2018 21:11:20 +0000\n\ntron (0.7.7.0) trusty; urgency=medium\n\n * 0.7.7.0 tagged with 'make release'\n * Cache job names in tab completion\n * Remove core service class (services are deprecated now)\n * Add backward compatible shelve\n * Add initial config fields for actions on PaaSTA\n * Bug fixes in tronfig and DST time resolution\n\n -- root Thu, 22 Mar 2018 18:33:40 +0000\n\ntron (0.7.6.1) trusty; urgency=medium\n\n * 0.7.6.1 tagged with 'make release'\n Commit: bump version, fix package building issues\n\n -- root Mon, 12 Mar 2018 17:08:34 +0000\n\ntron (0.7.6.0) trusty; urgency=medium\n\n * 0.7.6.0 tagged with 'make release'\n Commit: version bump 0.7.6.0\n\n -- root Mon, 12 Mar 2018 15:22:58 +0000\n\ntron (0.7.5.3) trusty; urgency=medium\n\n * 0.7.5.3 tagged with 'make release'\n Commit: Merge pull request #372 from Yelp/u/jgl/TRON-\n 212_upgrade_to_argparse Upgrade to argparse\n\n -- root Thu, 08 Mar 2018 00:39:13 +0000\n\ntron (0.7.5.2) trusty; urgency=medium\n\n * 0.7.5.2 tagged with 'make release'\n Commit: Merge pull request #373 from Yelp/fix_tz_naive_localization\n Only localize datetimes when they lack tzinfo\n\n -- root Fri, 02 Mar 2018 20:49:06 +0000\n\ntron (0.7.5.1) trusty; urgency=medium\n\n * 0.7.5.1 tagged with 'make release'\n Commit: Merge pull request #367 from\n Yelp/u/jgl/better_pidfile_error_message Make tron pidfile error\n message more clear\n\n -- root Wed, 28 Feb 2018 23:12:22 +0000\n\ntron (0.7.5.0) trusty; urgency=medium\n\n * 0.7.5.0 tagged with 'make release'\n Commit: Merge pull request #360 from Yelp/per_job_tz Allow jobs to\n override the default timezone\n\n -- root Wed, 28 Feb 2018 02:07:09 +0000\n\ntron (0.7.4.2) trusty; urgency=medium\n\n * 0.7.4.2 tagged with 'make release'\n Commit: added xenial building support\n\n -- root Tue, 27 Feb 2018 01:23:33 +0000\n\ntron (0.7.4.1) trusty; urgency=medium\n\n * 0.7.4.1 tagged with 'make release'\n Commit: Released 0.7.4.1 via make release\n\n -- root Fri, 23 Feb 2018 23:35:28 +0000\n\ntron (0.7.4.0) trusty; urgency=medium\n\n * 0.7.4.0 tagged with 'make release'\n * Remove support for mongodb in state serialization\n * Remove deprecated restart_interval option for services\n * Fix unicode bug in root URL\n\n -- root Tue, 13 Feb 2018 20:57:16 +0000\n\ntron (0.7.3.2) trusty; urgency=medium\n\n * 0.7.3.2 tagged with 'make release'\n Commit: Merge pull request #347 from Yelp/twisted-twisted Twisted\n fix, example cluster and itest improvements\n\n -- root Fri, 09 Feb 2018 14:39:45 +0000\n\ntron (0.7.3.1) trusty; urgency=medium\n\n * 0.7.3.1 tagged with 'make release'\n Commit: Merge branch dont-start-on-boot\n\n -- root Fri, 09 Feb 2018 09:27:12 +0000\n\ntron (0.7.3.0) trusty; urgency=medium\n\n * $0.7.3.0 tagged with \\make release'rCommit: Merge pull request\n\n -- root Thu, 08 Feb 2018 22:42:17 +0000\n\ntron (0.7.2.0) trusty; urgency=medium\n\n * 0.7.2.0 tagged with make release'\n * Use upstart instead of sysv-init\n * Added prototype check_tron_jobs and monitoring configs\n * Add --delete option for tronfig namespaces\n\n -- root Thu, 01 Feb 2018 11:07:08 +0000\n\ntron (0.7.1.0) trusty; urgency=medium\n\n * 0.7.1.0 tagged with 'make release'\n Commit: dont assume the USER env var use the more reliable\n getpass.getuser() instead of expecting a USER env var to be present.\n\n -- root Tue, 10 Oct 2017 12:47:50 +0000\n\ntron (0.7.0.0) trusty; urgency=medium\n\n * 0.7.0.0 tagged with 'make release'\n Commit: fix init.d script\n\n -- root Fri, 25 Aug 2017 14:22:47 +0000\n\ntron (0.6.2.1) lucid; urgency=low\n\n * Only keep last buffer from ssh connection\n\n -- Federico Giraud Mon, 15 Aug 2016 10:15:56 -0700\n\ntron (0.6.1.12) lucid; urgency=low\n\n * Fix memory leaks from event recorder and twisted\n\n -- Yejun Yang Wed, 06 Jan 2016 18:07:37 -0800\n\ntron (0.6.1.11) lucid; urgency=low\n\n * Add job and service support fields: owner, summary, notes\n\n -- Yejun Yang Fri, 02 Oct 2015 11:38:26 -0700\n\ntron (0.6.1.10) lucid; urgency=low\n\n * Optimize tronweb dashboard performance\n\n -- Yejun Yang Fri, 19 Dec 2014 13:45:23 -0800\n\ntron (0.6.1.9) lucid; urgency=low\n\n * Log some known exceptions\n * Check overlapped run id with instance\n\n -- Yejun Yang Thu, 11 Dec 2014 11:54:32 -0800\n\ntron (0.6.1.8) lucid; urgency=low\n\n * Fix service instance restore state, run monitor instead of queue.\n\n -- Yejun Yang Tue, 04 Nov 2014 14:14:29 -0800\n\ntron (0.6.1.7) lucid; urgency=low\n\n * Display error message when instance start fail\n\n -- Yejun Yang Tue, 04 Nov 2014 10:18:12 -0800\n\ntron (0.6.1.6) lucid; urgency=low\n\n * Ignore service instance start error\n * Ignore duplicated run id\n\n -- Yejun Yang Mon, 03 Nov 2014 17:21:24 -0800\n\ntron (0.6.1.5) lucid; urgency=low\n\n * Increase channel start timeout\n * Fix service monitor restart too soon\n\n -- Yejun Yang Thu, 03 Jul 2014 10:28:45 -0700\n\ntron (0.6.1.4) lucid; urgency=low\n\n * Remove incorrectly fixed dead code\n * Service monitor task always notify failed instead of down\n\n -- Yejun Yang Tue, 01 Apr 2014 10:43:48 -0700\n\ntron (0.6.1.3) lucid; urgency=low\n\n * Fix bug in node service stop\n\n -- Yejun Yang Thu, 27 Feb 2014 17:43:57 -0800\n\ntron (0.6.1.2) lucid; urgency=low\n\n * Fix bug prevent reconnection\n * Add new config monitor_retries\n\n -- Yejun Yang Wed, 05 Feb 2014 10:29:59 -0800\n\ntron (0.6.1) unstable; urgency=low\n\n * tronweb was replaced with a clientside version\n * more ssh options are now configurable\n * adding an experimental feature to support a max_runtime on jobs\n * adding tronctl kill to SIGKILL a service\n * add a `--no-header` option to tronfig\n\n -- Daniel Nephin Thu, 02 May 2013 17:34:46 -0700\n\ntron (0.6.0.2) unstable; urgency=low\n\n * Allow serviceinstances to transition from unknown to down\n * Better handling for serviceinstance monitor task failing\n\n -- Daniel Nephin Thu, 04 Apr 2013 12:47:14 -0700\n\ntron (0.6.0.1) unstable; urgency=low\n\n * minor visual improvements to tronview\n\n -- Daniel Nephin Tue, 26 Mar 2013 12:22:38 -0700\n\ntron (0.6.0) unstable; urgency=low\n\n * action.requires must be a list (string has been deprecated since 0.3.3)\n * tronctl zap has been removed (it shouldn't be necessary anymore)\n * service monitoring code has been re-written (services should not longer get stuck in a stopping state)\n * hosts can not be validated by specifying a known_hosts file\n * additional validation for ssh options and context variables has been moved into configuration validation\n * tronview now displays additional details about jobs and services\n\n -- Daniel Nephin Mon, 25 Mar 2013 11:14:13 -0700\n\ntron (0.5.2.3) unstable; urgency=low\n\n * Fix a bug that was preventing nodes from connecting with provided username\n * Patched an issue with the SSH connection that could cause exceptions on channel close\n\n -- Daniel Nephin Fri, 15 Feb 2013 11:19:25 -0800\n\ntron (0.5.2) unstable; urgency=low\n\n * Tron now supports the ability to use different users per node connection.\n * Fragmented configuration is now possible by using namespaced config files.\n * Additional cleanup and stability patches have been applied.\n * State persistence configuration can now be changed without restarting trond\n * State saving now includes a namespace, you will need to run `tools/migration/migrate_state.py` to migrate old state.\n\n -- Thomas Robinson Wed, 9 Jan 2013 16:25:53 -0700\n\ntron (0.5.1) unstable; urgency=low\n\n * Jobs which are disabled will no longer be re-enabled when part of their configuration changes.\n * Individual actions for a Job can no longer be started independently before a job is started. This was never intentionally supported.\n * Adding a new configuration option `allow_overlap` for Jobs, which allows job runs to overlap each other.\n * Jobs can now be configured using crontab syntax.\n\n -- Daniel Nephin Wed, 25 Jul 2012 16:25:53 -0700\n\ntron (0.5.0.2) unstable; urgency=low\n\n * Fix a bug with daemonizing and some versions of twisted reactor.\n\n -- Daniel Nephin Tue, 17 Jul 2012 19:21:39 -0700\n\ntron (0.5.0) unstable; urgency=low\n\n * Names for nodes, jobs, actions and service can now contain underscore characters but are restricted to 255 characters.\n * trond now supports a graceful shutdown. Send trond SIGINT to have it wait for all currently running jobs to complete before shutting down. SIGTERM also performs some cleanup before terminating.\n * State serialization has changed. See :ref:`config_state` for configuration options. `tools/migration/migrate_state.py` is included to migrate your existing Tron state to a new store. YAML store is now deprecated.\n * Old style config, which was deprecated in 0.3 will no longer work.\n\n -- Daniel Nephin Tue, 05 Jun 2012 18:47:34 -0700\n\ntron (0.4.1) unstable; urgency=low\n\n * tronview will once again attempt to find the tty width even when stdout is not a tty.\n * Fixed last_success for job context.\n * Job runs which are manually cancelled will now continue to schedule new runs.\n\n -- Daniel Nephin Wed, 30 May 2012 16:35:44 -0700\n\ntron (0.4.0) unstable; urgency=low\n\n * Jobs now continue to run all possible actions after one of its actions fail\n * Enabling a disabled job now schedules the next run using current time instead of the last successful run (which could cause many runs to be scheduled in the past if the job had been disabled for a while)\n * Resolved many inconsistencies and bugs around Job scheduling.\n\n -- Daniel Nephin Fri, 11 May 2012 18:00:00 -0800\n\ntron (0.3.3-1) unstable; urgency=low\n\n * Remove logrotate script from debian packaging\n * Add logging.conf to debian packaging\n\n -- James Brown Thu, 19 Apr 2012 14:33:17 -0700\n\ntron (0.3.3) unstable; urgency=low\n\n * Adding a configuration migration script for porting 0.2.x configs to the new 0.3.x\n * Remove working_dir from the configuration and replace with output_stream_dir\n * Remove logging confiruation from the general config. Logging is now configured using python standaring logging\n\n -- Daniel Nephin Wed, 18 Apr 2012 18:00:00 -0800\n\ntron (0.3.2) unstable; urgency=low\n\n * Fixes a bug when there are multiple node pools\n * Adds more unit tests\n\n -- Daniel Nephin Wed, 11 Apr 2012 11:35:05 -0800\n\ntron (0.3.1) unstable; urgency=low\n\n * Bug fix release\n * Adding state diagrams to documentation\n\n -- Daniel Nephin Tue, 27 Mar 2012 11:35:05 -0800\n\ntron (0.3.0) unstable; urgency=low\n\n * !Tags, *references, and &anchors are now deprecated in the trond\n configuration file. Support will be removed for them in 0.5.\n * Adding an enabled option for jobs, so they can be configured as disabled by default\n * tron commands (tronview, tronfig, tronctl) now support a global config (defaults to /etc/tron/tron.yaml)\n * tronview will now pipe its output through 'less' if appropriate\n\n -- Daniel Nephin Mon, 19 Feb 2012 11:35:05 -0800\n\ntron (0.2.10-1) unstable; urgency=low\n\n * ssh_options is actually optional (sjohnson)\n * Cleanup actions no longer cause jobs using an interval scheduler to stop being scheduled if an action fails (sjohnson)\n * Failed actions can be skipped, causing dependent actions to run (dnephin)\n * Tests have been moved from test/ to tests/ (sjohnson)\n * Everything under tron/ web/ and bin/ should now pass pyflakes\n\n -- Daniel Nephin Fri, 17 Feb 2012 11:35:05 -0800\n\ntron (0.2.9-1) unstable; urgency=low\n\n * tronweb works and is documented (mowings-iseatz)\n * Daylight Savings Time behavior is more well-defined (sjohnson)\n * Jobs that fail after running over their next scheduled time are no longer forgotten (sjohnson)\n * Reconfiguring syslog no longer requires restarting trond to take effect (jbrown)\n\n -- Steve Johnson Mon, 6 Feb 2012 16:26:05 -0800\n\ntron (0.2.8.1-1) unstable; urgency=low\n\n * Set a meaningful Formatter when logging to syslog (jbrown)\n * Included prebuilt man pages in distribution so Sphinx isn't required to\n have them\n\n -- James Brown Mon, 12 Dec 2011 16:26:05 -0800\n\ntron (0.2.8-1) unstable; urgency=low\n\n * Now on PyPI (irskep)\n * New HTML documentation at http://packages.python.org/tron (irskep)\n * Cleanup actions: run a command after the success or failure of a job (irskep)\n * Logging to syslog with syslog_address config field (irskep)\n * \"zap\" command for services (irskep)\n * simplejson is no longer a dependency for Python 2.6 and up (irskep)\n * Fix weekday-specified jobs (mon, tues, ...) running a day late (irskep)\n * Fix services being allowed in jobs list and causing weird crashes (irskep)\n * Fix missing import in www.py (irskep)\n * Better resilience to subtlely bad tronfigs (jbrown)\n\n -- Steve Johnson Fri, 25 Nov 2011 23:27:00 -0400\n\ntron (0.2.7-1) unstable; urgency=low\n\n * Really fix date parsing (rhettg)\n * Revert instant service monitor so we wait a while before checking our services (rhettg)\n * Clean up some logging (rhettg)\n\n -- Rhett Garber Wed, 14 Sep 2011 16:23:00 -0700\n\ntron (0.2.6-2) unstable; urgency=low\n\n * Fix date parsing\n\n -- James Brown Wed, 14 Sep 2011 15:36:30 -0700\n\ntron (0.2.6-1) unstable; urgency=low\n\n * Support for functional testing. Fixes #49 (irskep)\n * Context variables for year, month and day. Fixes #57 (irskep)\n * Integrate Google App Engine Cron scheduling syntax. Fixes #71 (irskep)\n * Fix crash during service monitoring because of node connect failures. Fixes #77 (rhettg)\n * Make action runs explicitly not re-startable. Fixes #78 (rhettg)\n * Flush and fsync state file. Fixes #74 (rhettg)\n * Handle node disconnect while waiting for channel to start. Fixes #75 (rhettg)\n * Replace an aggressive assert with a log message for monitor inconsistency. Fixes #73 (rhettg)\n * Handle tronview event listing issue with garbage collected entites. Fixes #70 (rhettg)\n * Prevent SSH stampedes by delaying some node EXEC calls (rhettg)\n\n -- Rhett Garber Wed, 14 Sep 2011 14:30:15 -0700\n\n\ntron (0.2.5-1) unstable; urgency=low\n\n * Introduce event collection system (rhettg)\n * Fix a crash in rebuilding all services under certain reconfig scenarios. Fixes #67 (rhettg)\n * Fix potential service situation where monitors would stop running after failures (rhettg)\n * Additional logging around startup failures (rhettg)\n\n -- Rhett Garber Wed, 22 June 2011 13:24:00 -0800\n\ntron (0.2.4-1) unstable; urgency=low\n\n * Final tronfig fix for stdout/stdin behavior (rhettg)\n\n -- Rhett Garber Tue, 12 Apr 2011 10:50:00 -0800\n\ntron (0.2.3-2) unstable; urgency=low\n\n * Made tronfig work with non-interactive uploads again. (jbrown)\n\n -- James Brown Mon, 11 Apr 2011 22:06:56 -0700\n\ntron (0.2.3-1) unstable; urgency=low\n\n * Resolved an issue where tronfig via stdin wouldn't catch all errors. (rhettg)\n * Provided additional config time validation to catch bad configurations. (rhettg)\n\n -- Rhett Garber Thu, 7 Apr 2011 10:50:00 -0800\n\ntron (0.2.2-1) unstable; urgency=low\n\n * Resolved an issue where certain service reconfigurations would\n cause the service to be stuck in the DOWN state (rhettg)\n * Reworked service to keep consistant instance numbers across restarts\n and reconfigs (rhettg)\n\n -- Rhett Garber Wed, 23 Mar 2011 18:26:00 -0800\n\ntron (0.2.1-1) unstable; urgency=low\n\n * Resolve an issue where run_time wasn't set for manually started jobs (rhettg)\n * Support for multiple arguments to tronctl (for starting things in bulk) (rhettg)\n * Support for starting a job with a specific run_time (rhettg)\n * Resolved an issue where services, after a reconfig, wouldn't cause state changes (rhettg)\n * Updated man pages (rhettg)\n\n -- Rhett Garber Wed, 09 Feb 2011 15:20:00 -0800\n\ntron (0.2.0-1) unstable; urgency=low\n\n * New services system (rhettg)\n\n -- Rhett Garber Mon, 06 Feb 2011 15:15:00 -0800\n\n\ntron (0.1.10-1) unstable; urgency=low\n\n * Remove use of deprecated twisted timeout calls. Fixes #9 (rhettg)\n * Handle newer versions of twisted (rhettg)\n * Dynamic column widths in tronview and better overflow (ebaum)\n * Command now displayed in tronview for an action Fixes #32 (ebaum)\n * Respect tronview -n option for stdout/stderr output. Fixes #41 (ebaum)\n * Show warnings option for tronview. Fixes #46 (ebaum)\n * Suppress headers option for tronview. (ebaum)\n * Fix an issue where default empty config failed to apply. (rhettg)\n * Add versioning to both tron module, command and state file (rhettg)\n * Set umask on daemon to allow proper pid-file control (rhettg)\n * Fix issue with command context not propogating on live reconfigs. Fixes\n #53 (rhettg)\n\n\n -- Rhett Garber Fri, 14 Jan 2011 13:10:00 -0800\n\ntron (0.1.9-1) unstable; urgency=low\n\n * Fix issue with config changes causing previous job runs to be in an\n unstable state. #42 (ebaum)\n\n -- Rhett Garber Mon, 14 Dec 2010 14:12:00 -0800\n\ntron (0.1.8-1) unstable; urgency=low\n\n * Address issue with bad format strings in commands causing untold\n disasters. #45 (rhettg)\n\n -- Rhett Garber Mon, 06 Dec 2010 17:38:00 -0800\n\ntron (0.1.7-1) unstable; urgency=low\n\n * Improve log rotation scripts under Debian (jbrown)\n * Fix an issue where removing a job with a live reconfig caused the job not to actually be removed. #44 (rhettg)\n * Some logging changes to make debugging issues easier (rhettg)\n * Some cleanup and better error/delay handling around process control for state writing. (rhettg)\n\n -- Rhett Garber Mon, 22 Nov 2010 15:52:00 -0800\n\ntron (0.1.6-1) unstable; urgency=low\n\n * Fix issue with live reconfigs causing intervals to be skipped (rhettg)\n * Added log file re-opening on SIGHUP (fixes #37) (rhettg)\n * Fix some issues with cmp functions for jobs that caused incorrect\n reconfigs (#38) (mtytel)\n * Fix issue with manually starting all_node jobs/services (mtytel)\n\n -- Rhett Garber Fri, 15 Oct 2010 15:05:00 -0700\n\ntron (0.1.5-1) unstable; urgency=low\n\n * Fixed crash due to config bug where SSH options were sometimes missing (rhettg)\n * Tweaks to command line interface (rhettg)\n\n -- Rhett Garber Wed, 14 Sep 2010 10:14:00 -0700\n\ntron (0.1.4-1) UNRELEASED; urgency=low\n\n * Simpler default options and config for trond (rhettg)\n * Trond daemonizing for proper init.d start/stop behavior (rhettg)\n * Fixes to reduce state file writing (matthewtytel)\n * Better pre-validation for tronfig (matthewtytel)\n * Updates to man pages (matthewtytel)\n\n -- Rhett Garber Tue, 7 Sep 2010 16:51:00 -0700\n\ntron (0.1.3-3) UNRELEASED; urgency=low\n\n * Use /var/lib/tron/ for a working directory (roguelazer)\n * Fix bug in Node configuration with services (rhettg)\n\n -- James Brown Mon, 2 Sep 2010 11:35:00 -0700\n\ntron (0.1.3-2) UNRELEASED; urgency=low\n\n * No longer depend on libyaml\n\n -- James Brown Mon, 2 Sep 2010 11:00:46 -0700\n\ntron (0.1.3-1) UNRELEASED; urgency=low\n\n * Better debian packaging (roguelazer)\n * Cleaner configuration (rhettg)\n * SIGHUP handling for reconfiguration (matthewtytel)\n * Command Context (environment variables for command execution) (rhettg)\n * Show job duration, alphabetize job list and direct stdout/stderr access (matthewtytel)\n\n -- James Brown Mon, 30 Aug 2010 18:33:00 -0700\n\ntron (0.1.2) UNRELEASED; urgency=low\n\n * Services (matthewtytel)\n * Smarter node pools (run all nodes) (matthewtytel)\n * Randomized node pool selection (matthewtytel)\n\n -- Rhett Garber Thu, 19 Aug 2010 11:05:00 -0700\n\ntron (0.1.1) UNRELEASED; urgency=low\n\n * On the fly reconfiguration (matthewtytel)\n * Saving state (matthewtytel)\n * job enable/disable (matthewtytel)\n\n -- Rhett Garber Thu, 19 Aug 2010 11:05:00 -0700\n\ntron (0.1.0) UNRELEASED; urgency=low\n\n * Initial release. (Closes: #XXXXXX)\n\n -- Rhett Garber Tue, 23 Mar 2010 07:34:36 -0700\n" }, { "path": "debian/compat", "content": "10\n" }, { "path": "debian/control", "content": "Source: tron\nSection: admin\nPriority: optional\nMaintainer: Daniel Nephin \nBuild-Depends: debhelper (>= 7), python3.10-dev, libdb5.3-dev, libyaml-dev, libssl-dev, libffi-dev, dh-virtualenv\nStandards-Version: 3.8.3\n\nPackage: tron\nArchitecture: all\nHomepage: http://github.com/yelp/Tron\nDepends: bsdutils, python3.10, libdb5.3, libyaml-0-2, ${shlibs:Depends}, ${misc:Depends}\nDescription: Tron is a job scheduling, running and monitoring package.\n Designed to replace Cron for complex scheduling and dependencies.\n Provides:\n Centralized configuration for running jobs across multiple machines\n Dependencies on jobs and resources\n Monitoring of jobs\n" }, { "path": "debian/copyright", "content": "This package was debianized by Steve Johnson \n on Sat, 26 Nov 2011 15:13:00 -0400.\n\nIt was downloaded from http://github.com/yelp/Tron\n\nUpstream Author:\n\n Rhett Garber \n Matt Tytel \n\nCopyright:\n\n Copyright 2010 Yelp\n\nLicense:\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "debian/docs", "content": "LICENSE.txt\nREADME.md\n" }, { "path": "debian/install", "content": "tron/logging.conf var/lib/tron\ntronweb/ opt/venvs/tron/\n" }, { "path": "debian/pycompat", "content": "2\n" }, { "path": "debian/pyversions", "content": "2.5-2.6\n" }, { "path": "debian/rules", "content": "#!/usr/bin/make -f\n# -*- makefile -*-\n\nDH_VERBOSE := 1\n\n%:\n\tdh $@ --with python-virtualenv\n\n# do not call `make clean` as part of packaging\noverride_dh_auto_clean:\n\ttrue\n\noverride_dh_auto_build:\n\ttrue\n\n# do not call `make test` as part of packaging\noverride_dh_auto_test:\n\ttrue\n\noverride_dh_virtualenv:\n\techo $(PIP_INDEX_URL)\n\tdh_virtualenv --index-url $(PIP_INDEX_URL) \\\n\t --extra-pip-arg --trusted-host=169.254.255.254 \\\n\t --extra-pip-arg --only-binary=cryptography \\\n --python=/usr/bin/python3.10 \\\n --preinstall cython==0.29.36 \\\n --preinstall pip==24.3.1 \\\n --preinstall setuptools==65.5.1\n\t@echo patching k8s client lib for configuration class\n\tpatch debian/tron/opt/venvs/tron/lib/python3.10/site-packages/kubernetes/client/configuration.py contrib/patch-config-loggers.diff\noverride_dh_installinit:\n\tdh_installinit --noscripts\n" }, { "path": "debian/tron.conffiles", "content": "/var/lib/tron/logging.conf\n" }, { "path": "debian/tron.default", "content": "# Defaults for tron initscript\n# sourced by /etc/init.d/tron\n# installed at /etc/default/tron by the maintainer scripts\n\n#\n# This is a POSIX shell fragment\n#\n\n# Additional options that are passed to the Daemon.\nDAEMON_OPTS=\"--log-conf /var/lib/tron/logging.conf\"\n\nLISTEN_HOST=\"0.0.0.0\"\nLISTEN_PORT=\"8089\"\n\n# User the daemon will run as. Needs to have appropriate credentials to SSH into your working nodes.\n# You should take care in setting permissions appropriately for log and working directories on /var\nDAEMONUSER=\"\"\n\n# Enable this when you have configured tron to your liking.\nRUN=\"no\"\n" }, { "path": "debian/tron.dirs", "content": "var/lib/tron/\nvar/log/tron/\n" }, { "path": "debian/tron.example", "content": "sample_config.yaml\n" }, { "path": "debian/tron.links", "content": "opt/venvs/tron/bin/check_tron_jobs usr/bin/check_tron_jobs.py\nopt/venvs/tron/bin/tronctl usr/bin/tronctl\nopt/venvs/tron/bin/trond usr/bin/trond\nopt/venvs/tron/bin/tronfig usr/bin/tronfig\nopt/venvs/tron/bin/tronview usr/bin/tronview\nopt/venvs/tron/bin/generate_tron_tab_completion_cache usr/bin/generate_tron_tab_completion_cache\nopt/venvs/tron/bin/tronctl_tabcomplete.sh usr/share/bash-completion/completions/tronctl\nopt/venvs/tron/bin/tronview_tabcomplete.sh usr/share/bash-completion/completions/tronview\n" }, { "path": "debian/tron.manpages", "content": "docs/source/man/tronctl.1\ndocs/source/man/trond.8\ndocs/source/man/tronview.1\ndocs/source/man/tronfig.1\n" }, { "path": "debian/tron.postinst", "content": "#!/bin/sh -e\n#\n# Post-installation script for tron\n\n#DEBHELPER#\n\nexit 0\n" }, { "path": "debian/tron.service", "content": "[Unit]\nDescription=trond\nAfter=network.target\n# Attempt restarts indefinitely (If omitted, systemd attempts max 5x within StartLimitIntervalSec)\nStartLimitIntervalSec=0\nStartLimitBurst=0\n\n[Service]\nUser=tron\nEnvironmentFile=/etc/default/tron\nExecStartPre=/bin/bash -c 'if pgrep -x trond >/dev/null; then echo \"ERROR: trond process already running\" >&2; exit 1; fi'\nExecStart=/usr/bin/zk-flock -k 60 tron_master_${CLUSTER_NAME} \"/usr/bin/trond --lock-file=${LOCKFILE:-$PIDFILE} --working-dir=${WORKINGDIR} --host ${LISTEN_HOST} --port ${LISTEN_PORT} ${DAEMON_OPTS}\"\nExecStopPost=/usr/bin/logger -t tron_exit_status \"SERVICE_RESULT:${SERVICE_RESULT} EXIT_CODE:${EXIT_CODE} EXIT_STATUS:${EXIT_STATUS}\"\n# This is generally not recommended, but we need to not send SIGKILL to the child trond process and instead let the SIGTERM from zk-flock propagate down\nKillMode=process\nTimeoutStopSec=20\nRestart=always\n# Wait between restart attempts\nRestartSec=10\nLimitNOFILE=100000\n\n[Install]\nWantedBy=multi-user.target\n" }, { "path": "debian/tron.upstart", "content": "description \"trond\"\n\nstart on filesystem and (started networking)\nstop on shutdown\n\nrespawn\nkill timeout 20\n\nscript\n set -a\n if [ -f /etc/default/tron ] ; then\n . /etc/default/tron\n fi\n if [ \"x$RUN\" != \"xyes\" ]; then\n log_failure_msg \"$NAME disabled, please adjust the configuration to your needs \"\n log_failure_msg \"and then set RUN to 'yes' in /etc/default/$NAME to enable it.\"\n exit 0\n fi\n exec start-stop-daemon --start -c $DAEMONUSER --exec /usr/bin/trond -- $DAEMON_OPTS\nend script\n" }, { "path": "debian/watch", "content": "# Example watch control file for uscan\n# Rename this file to \"watch\" and then you can run the \"uscan\" command\n# to check for upstream updates and more.\n# See uscan(1) for format\n\n# Compulsory line, this is a version 3 file\nversion=3\n\nhttp://githubredir.debian.net/github/Yelp/Tron\n" }, { "path": "dev/config/MASTER.yaml", "content": "# Please visit y/tron-development for a guide on how to setup Tron for local development\nstate_persistence:\n name: \"tron_state\"\n table_name: \"tmp-tron-state\"\n store_type: \"dynamodb\"\n buffer_size: 1\n dynamodb_region: us-west-1\n\neventbus_enabled: True\nssh_options:\n agent: True\n\nnodes:\n - hostname: localhost\n\n# Replace this with the path relative to your home dir to use\n# action_runner:\n# runner_type: \"subprocess\"\n# remote_status_path: \"pg/tron/status\"\n# remote_exec_path: \"pg/tron/.tox/py310/bin\"\n\njobs:\n testjob0:\n enabled: true\n node: localhost\n schedule: \"cron * * * * *\"\n run_limit: 5\n actions:\n zeroth:\n command: env\n trigger_downstreams:\n minutely: \"{ymdhm}\"\n cpus: 1\n mem: 100\n\n testjob1:\n enabled: false\n node: localhost\n schedule: \"cron * * * * *\"\n actions:\n first:\n command: \"sleep 5\"\n cpus: 1\n mem: 100\n second:\n command: \"echo 'hello world'\"\n requires: [first]\n triggered_by:\n - \"MASTER.testjob0.zeroth.minutely.{ymdhm}\"\n trigger_downstreams:\n minutely: \"{ymdhm}\"\n cpus: 1\n mem: 100\n\n testjob2:\n enabled: false\n node: localhost\n schedule: \"cron * * * * *\"\n actions:\n first:\n command: \"echo 'goodbye, world'\"\n cpus: 1\n mem: 100\n triggered_by:\n - \"MASTER.testjob1.second.minutely.{ymdhm}\"\n\n retrier:\n node: localhost\n schedule: \"cron 0 0 1 1 *\"\n actions:\n failing:\n command: exit 1\n retries: 1\n retries_delay: 5m\n" }, { "path": "dev/config/_manifest.yaml", "content": "MASTER: config/MASTER.yaml\n" }, { "path": "dev/logging.conf", "content": "[loggers]\nkeys=root, twisted, tron, tron.serialize.runstate.statemanager, tron.api.www.access, task_processing, tron.mesos.task_output, pymesos\n\n[handlers]\nkeys=stdoutHandler, accessHandler, nullHandler\n\n[formatters]\nkeys=defaultFormatter, accessFormatter\n\n[logger_root]\nlevel=WARN\nhandlers=stdoutHandler\n\n[logger_twisted]\nlevel=WARN\nhandlers=stdoutHandler\nqualname=twisted\npropagate=0\n\n[logger_tron]\nlevel=DEBUG\nhandlers=stdoutHandler\nqualname=tron\npropagate=0\n\n[logger_tron.api.www.access]\nlevel=DEBUG\nhandlers=accessHandler\nqualname=tron.api.www.access\npropagate=0\n\n[logger_tron.serialize.runstate.statemanager]\nlevel=DEBUG\nhandlers=stdoutHandler\nqualname=tron.serialize.runstate.statemanager\npropagate=0\n\n[logger_task_processing]\nlevel=INFO\nhandlers=stdoutHandler\nqualname=task_processing\npropagate=0\n\n[logger_pymesos]\nlevel=DEBUG\nhandlers=stdoutHandler\nqualname=pymesos\npropagate=0\n\n[logger_tron.mesos.task_output]\nlevel=INFO\nhandlers=nullHandler\nqualname=tron.mesos.task_output\npropagate=0\n\n[handler_stdoutHandler]\nclass=logging.StreamHandler\nlevel=DEBUG\nformatter=defaultFormatter\nargs=()\n\n[handler_nullHandler]\nclass=logging.NullHandler\nlevel=DEBUG\nargs=()\n\n[handler_accessHandler]\nclass=logging.StreamHandler\nlevel=DEBUG\nformatter=accessFormatter\nargs=()\n\n[formatter_defaultFormatter]\nformat=%(asctime)s %(name)s %(levelname)s %(message)s\n\n[formatter_accessFormatter]\nformat=%(message)s\n" }, { "path": "docs/source/_static/nature.css", "content": "/*\n * nature.css_t\n * ~~~~~~~~~~~~\n *\n * Sphinx stylesheet -- nature theme.\n *\n * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS.\n * :license: BSD, see LICENSE for details.\n *\n */\n\n@import url(\"basic.css\");\n\n/* -- page layout ----------------------------------------------------------- */\n\nbody {\n font-family: Arial, sans-serif;\n font-size: 100%;\n background-color: #111;\n color: #555;\n margin: 0;\n padding: 0;\n}\n\ndiv.documentwrapper {\n float: left;\n width: 100%;\n}\n\ndiv.bodywrapper {\n margin: 0 0 0 230px;\n}\n\nhr {\n border: 1px solid #B1B4B6;\n}\n\ndiv.document {\n background-color: #eee;\n}\n\ndiv.body {\n background-color: #ffffff;\n color: #3E4349;\n padding: 0 30px 30px 30px;\n font-size: 0.9em;\n}\n\ndiv.footer {\n color: #555;\n width: 100%;\n padding: 13px 0;\n text-align: center;\n font-size: 75%;\n}\n\ndiv.footer a {\n color: #444;\n text-decoration: underline;\n}\n\ndiv.related {\n background-color: #C41200;\n line-height: 32px;\n color: #fff;\n text-shadow: 0px 1px 0 #444;\n font-size: 0.9em;\n}\n\ndiv.related a {\n color: #F3F3CC;\n}\n\ndiv.sphinxsidebar {\n font-size: 0.75em;\n line-height: 1.5em;\n}\n\ndiv.sphinxsidebarwrapper{\n padding: 20px 0;\n}\n\ndiv.sphinxsidebar h3,\ndiv.sphinxsidebar h4 {\n font-family: Arial, sans-serif;\n color: #222;\n font-size: 1.2em;\n font-weight: normal;\n margin: 0;\n padding: 5px 10px;\n background-color: #ddd;\n text-shadow: 1px 1px 0 white\n}\n\ndiv.sphinxsidebar h4{\n font-size: 1.1em;\n}\n\ndiv.sphinxsidebar h3 a {\n color: #444;\n}\n\n\ndiv.sphinxsidebar p {\n color: #888;\n padding: 5px 20px;\n}\n\ndiv.sphinxsidebar p.topless {\n}\n\ndiv.sphinxsidebar ul {\n margin: 10px 20px;\n padding: 0;\n color: #000;\n}\n\ndiv.sphinxsidebar a {\n color: #444;\n}\n\ndiv.sphinxsidebar input {\n border: 1px solid #ccc;\n font-family: sans-serif;\n font-size: 1em;\n}\n\ndiv.sphinxsidebar input[type=text]{\n margin-left: 20px;\n}\n\n/* -- body styles ----------------------------------------------------------- */\n\na {\n color: #005B81;\n text-decoration: none;\n}\n\na:hover {\n color: #E32E00;\n text-decoration: underline;\n}\n\ndiv.body h1,\ndiv.body h2,\ndiv.body h3,\ndiv.body h4,\ndiv.body h5,\ndiv.body h6 {\n font-family: Arial, sans-serif;\n background-color: #BED4EB;\n font-weight: normal;\n color: #212224;\n margin: 30px 0px 10px 0px;\n padding: 5px 0 5px 10px;\n text-shadow: 0px 1px 0 white\n}\n\ndiv.body h1 { border-top: 20px solid white; margin-top: 0; font-size: 200%; }\ndiv.body h2 { font-size: 150%; background-color: #C8D5E3; }\ndiv.body h3 { font-size: 120%; background-color: #D8DEE3; }\ndiv.body h4 { font-size: 110%; background-color: #D8DEE3; }\ndiv.body h5 { font-size: 100%; background-color: #D8DEE3; }\ndiv.body h6 { font-size: 100%; background-color: #D8DEE3; }\n\na.headerlink {\n color: #c60f0f;\n font-size: 0.8em;\n padding: 0 4px 0 4px;\n text-decoration: none;\n}\n\na.headerlink:hover {\n background-color: #c60f0f;\n color: white;\n}\n\ndiv.body p, div.body dd, div.body li {\n line-height: 1.5em;\n}\n\ndiv.admonition p.admonition-title + p {\n display: inline;\n}\n\ndiv.highlight{\n background-color: white;\n}\n\ndiv.note {\n background-color: #eee;\n border: 1px solid #ccc;\n}\n\ndiv.seealso {\n background-color: #ffc;\n border: 1px solid #ff6;\n}\n\ndiv.topic {\n background-color: #eee;\n}\n\ndiv.warning {\n background-color: #ffe4e4;\n border: 1px solid #f66;\n}\n\np.admonition-title {\n display: inline;\n}\n\np.admonition-title:after {\n content: \":\";\n}\n\npre {\n padding: 10px;\n background-color: White;\n color: #222;\n line-height: 1.2em;\n border: 1px solid #C6C9CB;\n font-size: 1.1em;\n margin: 1.5em 0 1.5em 0;\n -webkit-box-shadow: 1px 1px 1px #d8d8d8;\n -moz-box-shadow: 1px 1px 1px #d8d8d8;\n}\n\ntt {\n background-color: #ecf0f3;\n color: #222;\n /* padding: 1px 2px; */\n font-size: 1.1em;\n font-family: monospace;\n}\n\n.viewcode-back {\n font-family: Arial, sans-serif;\n}\n\ndiv.viewcode-block:target {\n background-color: #f4debf;\n border-top: 1px solid #ac9;\n border-bottom: 1px solid #ac9;\n}\n" }, { "path": "docs/source/command_context.rst", "content": "\n.. _built_in_cc:\n\nBuilt-In Command Context Variables\n==================================\n\nTron includes some built in command context variables that can be used in\ncommand configuration for actions.\n\nThese variables can be used in the command of an action, using Python's format syntax (``{}``).\n\nOnce rendered into the command, they will **not** change. This is especially important for datetime-based context variables. Once a run is constructed, the datetime-based variables are \"frozen\", and will not change, even if the job is retried, or rerun one week later.\n\nFor example::\n\n # myservice.yaml\n myjob:\n node: localhost\n actions:\n myaction1:\n command: \"Hello world! I'm {action} for job {name} running on {node}\"\n\nThe command would get rendered at job runtime to::\n\n Hello world! I'm myaction1 for myservice.myjob running on localhost\n\n\n**shortdate**\n Run date in ``YYYY-MM-DD`` format. Supports simple arithmetic of the\n form ``{shortdate+6}`` which returns a date 6 days in the future,\n ``{shortdate-2}`` which returns a date 2 days before the run date.\n NOTE: this takes into account the job's configured timezone, if any.\n\n**ym, ymd, ymdh, ymdhm**\n Same as ``shortdate`` but better granularity. Arithmetic works with most\n granular unit: ``ymdh+1`` is +1 hours, ``ymdhm+1`` is +1 minute.\n NOTE: this takes into account the job's configured timezone, if any.\n\n**year**\n Current year in ``YYYY`` format. Supports the same arithmetic operations\n as `shortdate`. For example, ``{year-1}`` would return the year previous\n to the run date.\n NOTE: this takes into account the job's configured timezone, if any.\n\n**month**\n Current month in `MM` format. Supports the same arithmetic operations\n as `shortdate`. For example, ``{month+2}`` would return 2 months in the\n future.\n NOTE: this takes into account the job's configured timezone, if any.\n\n**day**\n Current day in `DD` format. Supports the same arithmetic operations\n as `shortdate`. For example, ``{day+1}`` would return the day after the\n run date.\n NOTE: this takes into account the job's configured timezone, if any.\n\n**hour**\n Current hour in `HH` (0-23) format. Supports the same arithmetic operations\n as `shortdate`. For example, ``{hour+1}`` would return the hour after the\n run hour (mod 24).\n NOTE: this takes into account the job's configured timezone, if any.\n\n**unixtime**\n Current timestamp. Supports addition and subtraction of seconds. For\n example ``{unixtime+20}`` would return the timestamp 20 seconds after\n the jobs runtime.\n\n**daynumber**\n Current day number as an ordinal (datetime.toordinal()). Supports addition\n and subtraction of days. For example ``{daynumber-3}`` would be 3 days\n before the run date.\n NOTE: this takes into account the job's configured timezone, if any.\n\n**name**\n Name of the job (e.g. ``myservice.myjob``).\n\n**actionnname**\n The name of the action (e.g. ``myaction1``).\n\n**node**\n Hostname of the node the action is being run on (e.g. ``localhost``).\n\n**runid**\n Run ID of the job run (e.g. ``sample_job.23``)\n\n**cleanup_job_status**\n ``SUCCESS`` if all actions have succeeded when the cleanup action runs,\n ``FAILURE`` otherwise. ``UNKNOWN`` if used in an action other than the\n cleanup action.\n\n**last_success**\n The last successful run date (defaults to current date if there was no\n previous successful run). Supports date arithmetic using the form\n ``{last_success#shortdate-1}``.\n\n**manual**\n ``true`` if the job was run manually. ``false`` otherwise.\n Manual job runs are those runs launched via the ``tronctl start`` command (as opposed to those launched by the scheduler).\n This variable is useful changing the behavior when jobs are run manually, like adding more verbose logging::\n\n command: \"myjob --verbose={manual}\"\n\n**namespace**\n The namespace of the config where the job comes from. Often ``MASTER`` or ``servicename``.\n Usually matches the name of service where the code runs.\n For example, if the job name is ``myservice.mycooljob.1.myaction``, ``{namespace}`` would be rendered as ``myservice``.\n\n\nBuilt In Environment Variables\n==============================\n\nThe following environment variables are also in the process environment.\n\nThese can be used like a normal linux environment variable using ``$``, like ``$TRON_JOB_NAMESPACE`` will be expanded at runtime and replace by the appropriate string.\n\nNote: These are **different** that Tron Context Variables, which are referenced using python style f-strings (``{myvariable}``) and are \"rendered\" into the command only, and not available as normal environment variables.\n\nIn all examples here, imagine running tronview like ``tronview myservice.myjob.42.myaction``. The example variables represent aspects of that particular action:\n\n**TRON_JOB_NAMESPACE**\n This is the tron config namespace where the job lives. Example: ``myservice``.\n\n**TRON_JOB_NAME**\n This variable is the top level key in the tron configuration file, like ``myjob``.\n\n**TRON_RUN_NUM**\n This is the job run number. Example: ``42``.\n\n**TRON_ACTION**\n This is the action name of the particular job. Example: ``myaction``.\n" }, { "path": "docs/source/conf.py", "content": "#\n# Tron documentation build configuration file, created by\n# sphinx-quickstart on Mon Nov 7 18:05:54 2011.\n#\n# This file is execfile()d with the current directory set to its containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\nimport os\nimport sys\nfrom unittest.mock import MagicMock\n\nsys.path.insert(0, os.path.abspath(\"../..\"))\n\nimport tron # noqa\n\n\nclass Mock(MagicMock):\n @classmethod\n def __getattr__(cls, name):\n return MagicMock()\n\n\nMOCK_MODULES = [\"bsddb3\"]\nsys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)\n\n\n# -- General configuration -----------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\nneeds_sphinx = \"1.0\"\n\n# Add any Sphinx extension module names here, as strings. They can be extensions\n# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.\nextensions = [\"sphinx.ext.autodoc\", \"sphinx.ext.coverage\"]\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = [\"_templates\"]\n\n# The suffix of source filenames.\nsource_suffix = \".rst\"\n\n# The encoding of source files.\n# source_encoding = 'utf-8-sig'\n\n# The master toctree document.\nmaster_doc = \"index\"\n\n# General information about the project.\nproject = \"Tron\"\ncopyright = \"2011, Yelp, Inc.\"\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\nversion = tron.__version__\n# The full version, including alpha/beta/rc tags.\nrelease = tron.__version__\n\n\n# There are two options for replacing |today|: either, you set today to some\n# non-false value, then it is used:\n# today = ''\n# Else, today_fmt is used as the format for a strftime call.\n# today_fmt = '%B %d, %Y'\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\nexclude_patterns = [\"_build\"]\n\n# The reST default role (used for this markup: `text`) to use for all documents.\n# default_role = None\n\n# If true, '()' will be appended to :func: etc. cross-reference text.\n# add_function_parentheses = True\n\n# If true, the current module name will be prepended to all description\n# unit titles (such as .. function::).\n# add_module_names = True\n\n# If true, sectionauthor and moduleauthor directives will be shown in the\n# output. They are ignored by default.\n# show_authors = False\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = \"sphinx\"\n\n# A list of ignored prefixes for module index sorting.\n# modindex_common_prefix = []\n\n\n# -- Options for HTML output ---------------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\nhtml_theme = \"nature\"\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n# html_theme_options = {}\n\n# Add any paths that contain custom themes here, relative to this directory.\n# html_theme_path = []\n\n# The name for this set of Sphinx documents. If None, it defaults to\n# \" v documentation\".\n# html_title = None\n\n# A shorter title for the navigation bar. Default is the same as html_title.\n# html_short_title = None\n\n# The name of an image file (relative to this directory) to place at the top\n# of the sidebar.\n# html_logo = None\n\n# The name of an image file (within the static path) to use as favicon of the\n# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32\n# pixels large.\n# html_favicon = None\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = [\"_static\"]\n\n# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,\n# using the given strftime format.\n# html_last_updated_fmt = '%b %d, %Y'\n\n# If true, SmartyPants will be used to convert quotes and dashes to\n# typographically correct entities.\n# html_use_smartypants = True\n\n# Custom sidebar templates, maps document names to template names.\n# html_sidebars = {}\n\n# Additional templates that should be rendered to pages, maps page names to\n# template names.\n# html_additional_pages = {}\n\n# If false, no module index is generated.\n# html_domain_indices = True\n\n# If false, no index is generated.\n# html_use_index = True\n\n# If true, the index is split into individual pages for each letter.\n# html_split_index = False\n\n# If true, links to the reST sources are added to the pages.\n# html_show_sourcelink = True\n\n# If true, \"Created using Sphinx\" is shown in the HTML footer. Default is True.\n# html_show_sphinx = True\n\n# If true, \"(C) Copyright ...\" is shown in the HTML footer. Default is True.\n# html_show_copyright = True\n\n# If true, an OpenSearch description file will be output, and all pages will\n# contain a tag referring to it. The value of this option must be the\n# base URL from which the finished HTML is served.\n# html_use_opensearch = ''\n\n# This is the file name suffix for HTML files (e.g. \".xhtml\").\n# html_file_suffix = None\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = \"Trondoc\"\n\n\n# -- Options for LaTeX output --------------------------------------------------\n\nlatex_elements = {\n # The paper size ('letterpaper' or 'a4paper').\n #'papersize': 'letterpaper',\n # The font size ('10pt', '11pt' or '12pt').\n #'pointsize': '10pt',\n # Additional stuff for the LaTeX preamble.\n #'preamble': '',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title, author, documentclass [howto/manual]).\nlatex_documents = [\n (\n \"index\",\n \"Tron.tex\",\n \"Tron Documentation\",\n \"Yelp, Inc.\",\n \"manual\",\n ),\n]\n\n# The name of an image file (relative to this directory) to place at the top of\n# the title page.\n# latex_logo = None\n\n# For \"manual\" documents, if this is true, then toplevel headings are parts,\n# not chapters.\n# latex_use_parts = False\n\n# If true, show page references after internal links.\n# latex_show_pagerefs = False\n\n# If true, show URL addresses after external links.\n# latex_show_urls = False\n\n# Documents to append as an appendix to all manuals.\n# latex_appendices = []\n\n# If false, no module index is generated.\n# latex_domain_indices = True\n\n\n# -- Options for manual page output --------------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [\n (\n \"man_tronview\",\n \"tronview\",\n \"tronview documentation\",\n [\"Yelp, Inc.\"],\n 1,\n ),\n (\n \"man_tronfig\",\n \"tronfig\",\n \"tronfig documentation\",\n [\"Yelp, Inc.\"],\n 1,\n ),\n (\n \"man_tronctl\",\n \"tronctl\",\n \"control Tron jobs\",\n [\"Yelp, Inc.\"],\n 1,\n ),\n (\n \"man_trond\",\n \"trond\",\n \"trond documentation\",\n [\"Yelp, Inc.\"],\n 8,\n ),\n]\n\n# If true, show URL addresses after external links.\n# man_show_urls = False\n\n\n# -- Options for Texinfo output ------------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (\n \"index\",\n \"Tron\",\n \"Tron Documentation\",\n \"Yelp, Inc.\",\n \"Tron\",\n \"One line description of project.\",\n \"Miscellaneous\",\n ),\n]\n\n# Documents to append as an appendix to all manuals.\n# texinfo_appendices = []\n\n# If false, no module index is generated.\n# texinfo_domain_indices = True\n\n# How to display URL addresses: 'footnote', 'no', or 'inline'.\n# texinfo_show_urls = 'footnote'\n" }, { "path": "docs/source/config.rst", "content": "Configuration\n=============\n\n.. _config_syntax:\n\nSyntax\n------\n\nThe Tron configuration file uses YAML syntax. The recommended configuration\nstyle requires only strings, decimal values, lists, and dictionaries: the\nsubset of YAML that can be losslessly transformed into JSON. (In fact, your\nconfiguration can be entirely JSON, since YAML is mostly a strict superset\nof JSON.)\n\nPast versions of Tron used additional YAML-specific features such as tags,\nanchors, and aliases. These features still work in version 0.3, but are now\ndeprecated.\n\nBasic Example\n-------------\n\n::\n\n ssh_options:\n agent: true\n\n nodes:\n - name: local\n hostname: 'localhost'\n\n jobs:\n \"getting_node_info\":\n node: local\n schedule: \"cron */10 * * * *\"\n actions:\n \"uname\":\n command: \"uname -a\"\n \"cpu_info\":\n command: \"cat /proc/cpuinfo\"\n requires: [uname]\n\n.. _command_context_variables:\n\nCommand Context Variables\n-------------------------\n\n**command** attribute values may contain **command context variables** that are\ninserted at runtime. The **command context** is populated both by Tron (see\n:ref:`built_in_cc`) and by the config file (see :ref:`command_context`). For\nexample::\n\n jobs:\n \"command_context_demo\":\n node: local\n schedule: \"1st monday in june\"\n actions:\n \"print_run_id\":\n # prints 'command_context_demo.1' on the first run,\n # 'command_context_demo.2' on the second, etc.\n command: \"echo {runid}\"\n\nSSH\n---\n\n**ssh_options** (optional)\n Options for SSH connections to Tron nodes. When tron runs a job\n on a node, it will add some jitter (random delay) to the run, which can be\n configured with the options below.\n\n **agent** (optional, default ``False``)\n Set to ``True`` if :command:`trond` should use an SSH agent. This requires\n that ``$SSH_AUTH_SOCK`` exists in the environment and points to the\n correct socket.\n\n **identities** (optional, default ``[]``)\n List of paths to SSH identity files\n\n **known_hosts_file** (optional, default ``None``)\n The path to an ssh known hosts file\n\n **connect_timeout** (optional, default ``30``)\n Timeout in seconds when establishing an ssh connection\n\n **idle_connection_timeout** (optional, default ``3600``)\n Timeout in seconds that an ssh connection can remain idle after which\n it is closed\n\n **jitter_min_load** (optional, default ``4``)\n Minimum `load` on a node before any jitter is introduced. See\n `jitter_load_factor` for a description of how load is calculated\n\n **jitter_max_delay** (optional, default ``20``)\n Maximum number of seconds to add to a run\n\n **jitter_load_factor** (optional, default ``1``)\n Factor used to increment the count of running actions for determining\n the upper bound of jitter to add (ex. A factor of 2 would increase the\n upper bound by 2 seconds per running action)\n\nExample::\n\n ssh_options:\n agent: false\n known_hosts_file: /etc/ssh/known_hosts\n identities:\n - /home/batch/.ssh/id_dsa-nopasswd\n\n connect_timeout: 30\n idle_connection_timeout: 3600\n\n jitter_min_load: 4\n jitter_max_delay: 20\n jitter_load_factor: 1\n\n.. _time_zone:\n\nTime Zone\n---------\n\n**time_zone** (optional)\n Local time as observed by the system clock. If your system is obeying a\n time zone with daylight savings time, then some of your jobs may run early\n or late on the days bordering each mode. See :ref:`dst_notes` for more\n information.\n\nExample::\n\n time_zone: US/Pacific\n\n.. _command_context:\n\nCommand Context\n---------------\n\n**command_context**\n Dictionary of custom :ref:`command context variables\n `. It is an arbitrary set of key-value pairs.\n\nExample::\n\n command_context:\n PYTHON: /usr/bin/python\n TMPDIR: /tmp\n\nSee a list of :ref:`built_in_cc`.\n\n\nOutput Stream Directory\n-----------------------\n**output_stream_dir**\n A path to the directory used to store the stdout/stderr logs from jobs.\n It defaults to the ``--working_dir`` option passed to :ref:`trond`.\n\nExample::\n\n output_stream_dir: \"/home/tronuser/output/\"\n\n\n.. _config_state:\n\nState Persistence\n-----------------\n**state_persistence**\n Configure how trond should persist its state to disk. By default a `shelve`\n store is used and saved to `./tron_state` in the working directory.\n\n **store_type**\n Valid options are:\n **shelve** - uses the `shelve` module and saves to a local file\n\n **yaml** - uses `yaml` and saves to a local file (this is not recommend and is provided to be backwards compatible with previous versions of Tron).\n\n You will need the appropriate python module for the option you choose.\n\n **name**\n The name of this store. This will be the filename for a **shelve** or\n **yaml** store.\n\n **buffer_size**\n The number of save calls to buffer before writing the state. Defaults to 1,\n which is no buffering.\n\n\nExample::\n\n state_persistence:\n store_type: shelve\n name: tron_store\n\n\n.. _action_runners:\n\nAction Runners\n--------------\n\n**Note:** this is an experimental feature\n\n\n**action_runner**\n Action runner configuration allows you to run Job actions through a script\n which records it's pid. This provides support for a max_runtime option\n on jobs, and allows you to stop or kill the action from :command:`tronctl`.\n\n **runner_type**\n Valid options are:\n **none**\n Run actions without a wrapper. This is the default\n\n **subprocess**\n Run actions with a script which records the pid and runs the\n action command in a subprocess (on the remote node). This\n requires that :command:`bin/action_runner.py` and\n :command:`bin/action_status.py` are available on the remote\n host.\n\n **remote_status_path**\n Path used to store status files. Defaults to `/tmp`.\n\n **remote_exec_path**\n Directory path which contains :command:`action_runner.py` and\n :command:`action_status.py` scripts.\n\n\nExample::\n\n action_runner:\n runner_type: \"subprocess\"\n remote_status_path: \"/tmp/tron\"\n remote_exec_path: \"/usr/local/bin\"\n\n\nNodes\n-----\n\n**nodes**\n List of nodes. Each node has the following options:\n\n **hostname** (required)\n The hostname or IP address of the node\n\n **name** (optional, defaults to ``hostname``)\n A name to refer to this node\n\n **username** (optional, defaults to current user)\n The name of the user to connect with\n\n **port** (optional, defaults to 22)\n The port number of the node\n\n\nExample::\n\n nodes:\n - name: node1\n hostname: 'batch1'\n - hostname: 'batch2' # name is 'batch2'\n\nNode Pools\n----------\n\n**node_pools**\n List of node pools, each with a ``name`` and ``nodes`` list. ``name``\n defaults to the names of each node joined by underscores.\n\nExample::\n\n node_pools:\n - name: pool\n nodes: [node1, batch1]\n - nodes: [batch1, node1] # name is 'batch1_node1'\n\nJobs and Actions\n----------------\n\n**jobs**\n List of jobs for Tron to manage. See :doc:`jobs` for the options available\n to jobs and their actions.\n\n.. _config_logging:\n\nLogging\n-------\n\nAs of v0.3.3 Logging is no longer configured in the tron configuration file.\n\nTron uses Python's standard logging and by default uses a rotating log file\nhandler that rotates files each day. The default log directory is\n``/var/log/tron/tron.log``.\n\nTo configure logging pass -l to trond. You can modify the\ndefault logging.conf by copying it from tron/logging.conf. See\nhttp://docs.python.org/howto/logging.html#configuring-logging\n\nInteresting logs\n~~~~~~~~~~~~~~~~\n\nMost tron logs are named by using pythons `__file__` which uses the modules\nname. There are a couple special cases:\n\n**twisted**\n Twisted sends its logs to the `twisted` log\n\n**tron.api.www.access**\n API access logs are sent to this log at the INFO log level. They follow\n a standard apache combined log format.\n" }, { "path": "docs/source/developing.rst", "content": ".. _developing:\n\nContributing to Tron\n====================\n\nTron is an open source project and welcomes contributions from the community.\nThe source and issue tracker are hosted on github at\nhttp://github.com/yelp/Tron.\n\nSetting Up an Environment\n-------------------------\n\nTron works well with `virtualenv `_, which can be\nsetup using `virtualenvwrapper\n`_::\n\n $ mkvirtualenv tron --distribute --no-site-packages\n $ pip install -r dev/req_dev.txt\n\n``req_dev.txt`` contains a list of packages required for development,\nto run the tests, and `Sphinx `_ to build the documentation.\n\nCoding Standards\n----------------\n\nAll code should be `PEP8 `_ compliant,\nand should pass pyflakes without warnings. All new code should include full\ntest coverage, and bug fixes should include a test which reproduces the\nreported issue.\n\nThis documentation must also be kept up to date with any changes in functionality.\n\n\nRunning Tron in a Sandbox\n-------------------------\n\nThe source package includes a development logging.conf and a\nsample configuration file with a few test cases. To run a development instance\nof Tron create a working directory and start\n:command:`trond` using the following::\n\n $ make dev\n\n\nRunning the Tests\n-----------------\nRun the tests using ``make test``.\n\nContributing\n------------\n\nThere should be a github issue created prior to all pull requests. Pull requests\nshould be made to the ``Yelp:development`` branch, and should include additions to\n``CHANGES.txt`` which describe what has changed.\n" }, { "path": "docs/source/generated/modules.rst", "content": "tron\n====\n\n.. toctree::\n :maxdepth: 4\n\n tron\n" }, { "path": "docs/source/generated/tron.actioncommand.rst", "content": "tron.actioncommand module\n=========================\n\n.. automodule:: tron.actioncommand\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.api.adapter.rst", "content": "tron.api.adapter module\n=======================\n\n.. automodule:: tron.api.adapter\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.api.async_resource.rst", "content": "tron.api.async\\_resource module\n===============================\n\n.. automodule:: tron.api.async_resource\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.api.auth.rst", "content": "tron.api.auth module\n====================\n\n.. automodule:: tron.api.auth\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.api.controller.rst", "content": "tron.api.controller module\n==========================\n\n.. automodule:: tron.api.controller\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.api.requestargs.rst", "content": "tron.api.requestargs module\n===========================\n\n.. automodule:: tron.api.requestargs\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.api.resource.rst", "content": "tron.api.resource module\n========================\n\n.. automodule:: tron.api.resource\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.api.rst", "content": "tron.api package\n================\n\nSubmodules\n----------\n\n.. toctree::\n :maxdepth: 4\n\n tron.api.adapter\n tron.api.async_resource\n tron.api.auth\n tron.api.controller\n tron.api.requestargs\n tron.api.resource\n\nModule contents\n---------------\n\n.. automodule:: tron.api\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.command_context.rst", "content": "tron.command\\_context module\n============================\n\n.. automodule:: tron.command_context\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.commands.authentication.rst", "content": "tron.commands.authentication module\n===================================\n\n.. automodule:: tron.commands.authentication\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.commands.backfill.rst", "content": "tron.commands.backfill module\n=============================\n\n.. automodule:: tron.commands.backfill\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.commands.client.rst", "content": "tron.commands.client module\n===========================\n\n.. automodule:: tron.commands.client\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.commands.cmd_utils.rst", "content": "tron.commands.cmd\\_utils module\n===============================\n\n.. automodule:: tron.commands.cmd_utils\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.commands.display.rst", "content": "tron.commands.display module\n============================\n\n.. automodule:: tron.commands.display\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.commands.retry.rst", "content": "tron.commands.retry module\n==========================\n\n.. automodule:: tron.commands.retry\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.commands.rst", "content": "tron.commands package\n=====================\n\nSubmodules\n----------\n\n.. toctree::\n :maxdepth: 4\n\n tron.commands.authentication\n tron.commands.backfill\n tron.commands.client\n tron.commands.cmd_utils\n tron.commands.display\n tron.commands.retry\n\nModule contents\n---------------\n\n.. automodule:: tron.commands\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.config.config_parse.rst", "content": "tron.config.config\\_parse module\n================================\n\n.. automodule:: tron.config.config_parse\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.config.config_utils.rst", "content": "tron.config.config\\_utils module\n================================\n\n.. automodule:: tron.config.config_utils\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.config.manager.rst", "content": "tron.config.manager module\n==========================\n\n.. automodule:: tron.config.manager\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.config.rst", "content": "tron.config package\n===================\n\nSubmodules\n----------\n\n.. toctree::\n :maxdepth: 4\n\n tron.config.config_parse\n tron.config.config_utils\n tron.config.manager\n tron.config.schedule_parse\n tron.config.schema\n tron.config.static_config\n\nModule contents\n---------------\n\n.. automodule:: tron.config\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.config.schedule_parse.rst", "content": "tron.config.schedule\\_parse module\n==================================\n\n.. automodule:: tron.config.schedule_parse\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.config.schema.rst", "content": "tron.config.schema module\n=========================\n\n.. automodule:: tron.config.schema\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.config.static_config.rst", "content": "tron.config.static\\_config module\n=================================\n\n.. automodule:: tron.config.static_config\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.core.action.rst", "content": "tron.core.action module\n=======================\n\n.. automodule:: tron.core.action\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.core.actiongraph.rst", "content": "tron.core.actiongraph module\n============================\n\n.. automodule:: tron.core.actiongraph\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.core.actionrun.rst", "content": "tron.core.actionrun module\n==========================\n\n.. automodule:: tron.core.actionrun\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.core.job.rst", "content": "tron.core.job module\n====================\n\n.. automodule:: tron.core.job\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.core.job_collection.rst", "content": "tron.core.job\\_collection module\n================================\n\n.. automodule:: tron.core.job_collection\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.core.job_scheduler.rst", "content": "tron.core.job\\_scheduler module\n===============================\n\n.. automodule:: tron.core.job_scheduler\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.core.jobgraph.rst", "content": "tron.core.jobgraph module\n=========================\n\n.. automodule:: tron.core.jobgraph\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.core.jobrun.rst", "content": "tron.core.jobrun module\n=======================\n\n.. automodule:: tron.core.jobrun\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.core.recovery.rst", "content": "tron.core.recovery module\n=========================\n\n.. automodule:: tron.core.recovery\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.core.rst", "content": "tron.core package\n=================\n\nSubmodules\n----------\n\n.. toctree::\n :maxdepth: 4\n\n tron.core.action\n tron.core.actiongraph\n tron.core.actionrun\n tron.core.job\n tron.core.job_collection\n tron.core.job_scheduler\n tron.core.jobgraph\n tron.core.jobrun\n tron.core.recovery\n\nModule contents\n---------------\n\n.. automodule:: tron.core\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.eventbus.rst", "content": "tron.eventbus module\n====================\n\n.. automodule:: tron.eventbus\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.kubernetes.rst", "content": "tron.kubernetes module\n======================\n\n.. automodule:: tron.kubernetes\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.manhole.rst", "content": "tron.manhole module\n===================\n\n.. automodule:: tron.manhole\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.mcp.rst", "content": "tron.mcp module\n===============\n\n.. automodule:: tron.mcp\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.mesos.rst", "content": "tron.mesos module\n=================\n\n.. automodule:: tron.mesos\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.metrics.rst", "content": "tron.metrics module\n===================\n\n.. automodule:: tron.metrics\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.node.rst", "content": "tron.node module\n================\n\n.. automodule:: tron.node\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.prom_metrics.rst", "content": "tron.prom\\_metrics module\n=========================\n\n.. automodule:: tron.prom_metrics\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.rst", "content": "tron package\n============\n\nSubpackages\n-----------\n\n.. toctree::\n :maxdepth: 4\n\n tron.api\n tron.commands\n tron.config\n tron.core\n tron.serialize\n tron.utils\n\nSubmodules\n----------\n\n.. toctree::\n :maxdepth: 4\n\n tron.actioncommand\n tron.command_context\n tron.eventbus\n tron.kubernetes\n tron.manhole\n tron.mcp\n tron.mesos\n tron.metrics\n tron.node\n tron.prom_metrics\n tron.scheduler\n tron.ssh\n tron.trondaemon\n tron.yaml\n\nModule contents\n---------------\n\n.. automodule:: tron\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.scheduler.rst", "content": "tron.scheduler module\n=====================\n\n.. automodule:: tron.scheduler\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.serialize.filehandler.rst", "content": "tron.serialize.filehandler module\n=================================\n\n.. automodule:: tron.serialize.filehandler\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.serialize.rst", "content": "tron.serialize package\n======================\n\nSubpackages\n-----------\n\n.. toctree::\n :maxdepth: 4\n\n tron.serialize.runstate\n\nSubmodules\n----------\n\n.. toctree::\n :maxdepth: 4\n\n tron.serialize.filehandler\n\nModule contents\n---------------\n\n.. automodule:: tron.serialize\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.serialize.runstate.dynamodb_state_store.rst", "content": "tron.serialize.runstate.dynamodb\\_state\\_store module\n=====================================================\n\n.. automodule:: tron.serialize.runstate.dynamodb_state_store\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.serialize.runstate.rst", "content": "tron.serialize.runstate package\n===============================\n\nSubmodules\n----------\n\n.. toctree::\n :maxdepth: 4\n\n tron.serialize.runstate.dynamodb_state_store\n tron.serialize.runstate.shelvestore\n tron.serialize.runstate.statemanager\n tron.serialize.runstate.yamlstore\n\nModule contents\n---------------\n\n.. automodule:: tron.serialize.runstate\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.serialize.runstate.shelvestore.rst", "content": "tron.serialize.runstate.shelvestore module\n==========================================\n\n.. automodule:: tron.serialize.runstate.shelvestore\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.serialize.runstate.statemanager.rst", "content": "tron.serialize.runstate.statemanager module\n===========================================\n\n.. automodule:: tron.serialize.runstate.statemanager\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.serialize.runstate.yamlstore.rst", "content": "tron.serialize.runstate.yamlstore module\n========================================\n\n.. automodule:: tron.serialize.runstate.yamlstore\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.ssh.rst", "content": "tron.ssh module\n===============\n\n.. automodule:: tron.ssh\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.trondaemon.rst", "content": "tron.trondaemon module\n======================\n\n.. automodule:: tron.trondaemon\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.utils.collections.rst", "content": "tron.utils.collections module\n=============================\n\n.. automodule:: tron.utils.collections\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.utils.crontab.rst", "content": "tron.utils.crontab module\n=========================\n\n.. automodule:: tron.utils.crontab\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.utils.exitcode.rst", "content": "tron.utils.exitcode module\n==========================\n\n.. automodule:: tron.utils.exitcode\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.utils.logreader.rst", "content": "tron.utils.logreader module\n===========================\n\n.. automodule:: tron.utils.logreader\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.utils.observer.rst", "content": "tron.utils.observer module\n==========================\n\n.. automodule:: tron.utils.observer\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.utils.persistable.rst", "content": "tron.utils.persistable module\n=============================\n\n.. automodule:: tron.utils.persistable\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.utils.proxy.rst", "content": "tron.utils.proxy module\n=======================\n\n.. automodule:: tron.utils.proxy\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.utils.queue.rst", "content": "tron.utils.queue module\n=======================\n\n.. automodule:: tron.utils.queue\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.utils.rst", "content": "tron.utils package\n==================\n\nSubmodules\n----------\n\n.. toctree::\n :maxdepth: 4\n\n tron.utils.collections\n tron.utils.crontab\n tron.utils.exitcode\n tron.utils.logreader\n tron.utils.observer\n tron.utils.persistable\n tron.utils.proxy\n tron.utils.queue\n tron.utils.state\n tron.utils.timeutils\n tron.utils.trontimespec\n tron.utils.twistedutils\n\nModule contents\n---------------\n\n.. automodule:: tron.utils\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.utils.state.rst", "content": "tron.utils.state module\n=======================\n\n.. automodule:: tron.utils.state\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.utils.timeutils.rst", "content": "tron.utils.timeutils module\n===========================\n\n.. automodule:: tron.utils.timeutils\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.utils.trontimespec.rst", "content": "tron.utils.trontimespec module\n==============================\n\n.. automodule:: tron.utils.trontimespec\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.utils.twistedutils.rst", "content": "tron.utils.twistedutils module\n==============================\n\n.. automodule:: tron.utils.twistedutils\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/generated/tron.yaml.rst", "content": "tron.yaml module\n================\n\n.. automodule:: tron.yaml\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "docs/source/index.rst", "content": "Tron\n====\n\nTron is a centralized system for managing periodic batch processes\nacross a cluster. If this is your first time using Tron, read :doc:`tutorial`\nand :doc:`overview` to get a better idea of what it is, how it works, and how\nto use it.\n\n.. note::\n\n Please report bugs in the documentation at `our Github issue tracker\n `_.\n\nTable of Contents\n-----------------\n\n.. toctree::\n :maxdepth: 2\n\n whats-new.rst\n tutorial.rst\n overview.rst\n config.rst\n jobs.rst\n command_context.rst\n tronweb.rst\n tools.rst\n developing.rst\n\nGenerated Docs\n~~~~~~~~~~~~~~\n\n.. toctree::\n :maxdepth: 1\n\n generated/modules\n\nIndices and tables\n==================\n\n* :ref:`genindex`\n* :ref:`search`\n" }, { "path": "docs/source/jobs.rst", "content": "Jobs\n====\n\nA job consists of a name, a node/node pool, a list of actions, a schedule, and\nan optional cleanup action. They are periodic events that do not interact with\nother jobs while running.\n\nIf all actions exit with status 0, the job has succeeded. If any action exists\nwith a nonzero status, the job has failed.\n\n\nRequired Fields\n---------------\n\nJobs are defined in the form of a dictionary, where the **name** is the key.\nThe Name of the job is used in :command:`tronview` and :command:`tronctl`. Here is an example::\n\n jobs:\n \"foo\":\n \"schedule\": ...\n \"command\": ...\n \"actions\":\n \"run_first\":\n \"command\": ...\n\n**node**\n Reference to the node or pool to run the job in. If a pool, the job is\n run in a random node in the pool.\n\n**schedule**\n When to run this job. Schedule fields can take multiple forms. See\n :ref:`job_scheduling`.\n\n**actions**\n List of :ref:`actions `.\n\nOptional Fields\n---------------\n\n**monitoring** (default **{}**)\n Dictionary of key: value pairs to inform the monitoring framework on how to\n alert teams for job failures.\n\n If you're using PaaSTA, for any monitoring fields not specified for a job,\n Tron will default to those set in `monitoring.yaml` in your soa-configs.\n\n You can see more about this behavior in the `PaaSTA docs`_.\n\n .. _PaaSTA docs: https://paasta.readthedocs.io/en/latest/yelpsoa_configs.html#monitoring-yaml\n\n **team**\n Team responsible for the job. Must already be defined in the monitoring\n framework.\n\n **page** (default **False**)\n Boolean on whether or not an alert for this job is page-worthy.\n\n **runbook**\n Runbook associated with the job.\n\n **tip** (default **None**)\n A short 1-line version of the runbook.\n\n **notification_email**\n A comma-separated string of email destinations. Defaults to the \"team\"\n default.\n\n **slack_channels**\n A list of Slack channels to send alerts to. Defaults to the team setting.\n Set an empty list to specify no Slack notifications.\n\n **ticket** (default **False**)\n A Boolean value to enable ticket creation.\n\n **project** (default **None**)\n A string representing the JIRA project that the ticket should go under.\n Defaults to the team value.\n\n **priority** (default **None**)\n A JIRA ticket priority to use when creating a ticket. This only makes\n sense to use when in combination with the ticket parameter set to True.\n This value should be a string value like '0', '1', '3.14', etc. If not\n set, the default will be the default_priority setting for the sensu team\n or the default priority used for the JIRA project.\n\n **tags** (default **None**)\n A list of arbitrary tags that can be used in handlers for different\n metadata needs.\n\n **component** (default **None**)\n A list of components affected by the event. A good example here would be\n to include the job that is being affected.\n\n **description** (default **None**)\n Human readable text giving more context on any monitoring events.\n\n **check_that_every_day_has_a_successful_run** (default **False**)\n If **True**, the latest job run each day will be checked to see if it was\n successful.\n\n If **False**, only the latest overall run will be checked to see if it was\n successful.\n\n **page_for_expected_runtime** (default **False**)\n If **True**, when either a job or an action exceeds its configured ``expected_runtime``, the generated alert will be considered \"critical\" and will page the user.\n\n If **False**, then an alert will not page the user.\n\n\n**queueing** (default **True**)\n If a job run is still running when the next job run is to be scheduled,\n add the next run to a queue if this is **True**. Otherwise, cancel\n the job run. Note that if the scheduler used for this job is\n not defined to queue overlapping then this setting is ignored.\n\n**allow_overlap** (default **False**)\n If **True** new job runs will start even if the previous run is still running.\n By default new job runs are either cancelled or queued (see **queuing**).\n\n**run_limit** (default **50**)\n Number of runs which will be stored. Once a Job has more then run_limit\n runs, the output and state for the oldest run are removed. Failed runs\n will not be removed.\n\n**all_nodes** (default **False**)\n If **True** run this job on each node in the\n node pool list. If a node appears more than once in the list, the job will\n be run on that node once for each appearance.\n\n If **False** run this job on a random node\n from the node pool list. If a node appears more than once in the list, the\n job will be more likely to run on that node, proportionate to the number of\n appearances.\n\n If **node** is not a node pool, this option has no effect.\n\n**cleanup_action**\n Action to run when either all actions have succeeded or the job has failed.\n See :ref:`job_cleanup_actions`.\n\n**enabled** (default **True**)\n If **False** the job will not be scheduled to run.\n\n**max_runtime** (default **None**)\n A time interval (ex: \"2 hours\") that limits the duration of each job run.\n If the job run is still running after this duration, all of its actions\n are sent SIGTERM.\n\n**time_zone** (default **None**)\n Time zone used for calculating when a job should run. Defaults to\n None, which means it will use the default time_zone set in the master\n config.\n\n**expected_runtime** (default **24h**)\n A time interval (ex: \"2 hours\") that specifies the maximum expected duration of each job run.\n Single units like (20m, 1h, 2d) are accepted, but you can't use mixed units like (1h 20m)\n Monitoring will alert if a job run is still running after this duration.\n Use max_runtime instead if hard limit is needed.\n\n\n.. _job_actions:\n\nActions\n-------\n\nActions consist primarily of a **command**. An action's command is\nexecuted as soon as its dependencies (specified by **requires**) are satisfied.\nSo if your job has 10 actions, 1 of which depends on the other 9, then Tron\nwill launch the first 9 actions in parallel and run the last one when all have\ncompleted successfully.\n\nIf any action exits with nonzero status, the job will continue to run any\nactions which do not depend on the failed action.\n\n\nRequired Fields\n^^^^^^^^^^^^^^^\n\nActions are defined as a dictionary, where the Name of the action is the key.\nThe Name is used in :command:`tronview` and :command:`tronctl`.\n\n**command**\n Command to run. Commands are run using ``/bin/sh`` so bash\n expressions will not work, and could cause the job to fail.\n\nOptional Fields\n^^^^^^^^^^^^^^^\n\n**requires**\n List of action names that must complete successfully before this\n action is run. Actions can only require actions in the same job.\n\n**node**\n Node or node pool to run the action on if different from the rest of the\n job.\n\n**retries**\n An integer representing how many times Tron is allowed to automatically\n retry the command. Tron will immediately re-run the command if it fails,\n and the action will not enter the failed state until retries are exhausted.\n Defaults to None (0 retries allowed).\n\n**retries_delay**\n A timedelta to wait in between retries.\n\n**expected_runtime** (default **24h**)\n A time interval (ex: \"2 hours\") that specifies the maximum expected duration\n of each action run. Monitoring will alert if a action run is still running\n after this duration.\n\n**trigger_downstreams** (bool or dict)\n Upon successful completion of an action, will emit a trigger for every\n item in the dictionary. When set to ``true``, a default dict of\n ``{shortdate: \"{shortdate}\"}`` is assumed. Emitted triggers will be in form:\n ``....``. See\n ``triggered_by`` for more information.\n\n**triggered_by** (list)\n When list is not empty, action will not start until all required triggers\n have been emitted by upstream actions. Unlike with ``requires`` attribute,\n dependent actions don't have to belong to the same job. ``triggered_by``\n template may contain any pattern allowed in ``command`` attribute.\n See :ref:`shortdate` for an explanation of shortdate\n\n Example:\n\n::\n triggered_by:\n - \"other_namespace.some_job.action1.shortdate.{shortdate-1}\"\n\n**trigger_timeout** (default **24h**)\n How long will action wait for dependencies listed in ``triggered_by`` before\n failing. Is not included in ``expected_runtime``. If upstream job fails, no\n trigger event will be emitted and downstream jobs will fail with trigger\n timeout. Re-running upstream job will emit the trigger upon successful\n completion and if any downstream job is still waiting - it will proceed\n normally. Timed out downstream jobs will not be re-started, and you need to\n use ``tronctl publish`` to trigger it manually. .\n\n\n\nExample Actions\n^^^^^^^^^^^^^^^\n\n::\n\n jobs:\n \"convert_logs\":\n node: node1\n schedule:\n start_time: 04:00:00\n actions:\n \"verify_logs_present\":\n command: \"ls /var/log/app/log_{shortdate-1}.txt\"\n \"convert_logs\":\n command: \"convert_logs /var/log/app/log_{shortdate-1}.txt /var/log/app_converted/log_{shortdate-1}.txt\"\n requires: [verify_logs_present]\n\n.. _job_scheduling:\n\nScheduling\n----------\n\nTron supports four methods for configuring the schedule of a job. Schedulers\nsupport a jitter parameter that allows them to vary their runtime by a\nrandom time delta.\n\n\nDaily\n^^^^^\n\nRun the job on specific days at a specific time. The time expression is\n``HH:MM:SS[ MTWRFSU]``.\n\nShort form::\n\n schedule: \"daily 04:00:00\"\n\nShort form with days::\n\n schedule: \"daily 04:00:00 MWF\"\n\nLong form::\n\n schedule:\n type: \"daily\"\n value: \"07:00:00 MWF\"\n jitter: \"10 min\" # Optional\n\nCron\n^^^^\n\nSchedule a job using cron syntax. Tron supports predefined schedules, ranges,\nand lists for each field. It supports the *L* in day of month field only (which\nschedules the job on the last day of the month). Only one of the day fields\n(day of month and day of week) can have a value.\n\n\nShort form::\n\n schedule: \"cron */5 * * 7,8 *\" # Every 5 minutes in July and August\n\n::\n\n schedule: \"cron 0 3-6 * * *\" # Every hour between 3am and 6am\n\nLong form::\n\n schedule: # long form\n type: \"cron\"\n value: \"30 4 L * *\" # The last day of the month at 4:30am\n\n\nComplex\n^^^^^^^\n\nMore powerful version of the daily scheduler based on the one used by Google\nApp Engine's cron library. To use this scheduler, use a string in this format\nas the schedule::\n\n (\"every\"|ordinal) (days) [\"of|in\" (monthspec)] ([\"at\"] HH:MM)\n\n**ordinal**\n Comma-separated list of ``1st`` and so forth. Use ``every`` if you don't want\n to limit by day of the month.\n\n**days**\n Comma-separated list of days of the week (for example, ``mon``, ``tuesday``,\n with both short and long forms being accepted); ``every day`` is equivalent\n to ``every mon,tue,wed,thu,fri,sat,sun``\n\n**monthspec**\n Comma-separated list of month names (for example, ``jan``, ``march``, ``sep``).\n If omitted, implies every month. You can also say ``month`` to mean every\n month, as in ``1,8th,15,22nd of month 09:00``.\n\n**HH:MM**\n Time of day in 24 hour time.\n\nSome examples::\n\n 2nd,third mon,wed,thu of march 17:00\n every monday at 09:00\n 1st monday of sep,oct,nov at 17:00\n every day of oct at 00:00\n\nIn the config::\n\n schedule: \"every monday at 09:00\"\n\n::\n\n schedule:\n type: \"groc daily\"\n value: \"every day 11:22\"\n jitter: \"5 min\"\n\n.. _dst_notes:\n\nNotes on Daylight Saving Time\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSome system clocks are configured to track local time and may observe daylight\nsavings time. For example, on November 6, 2011, 1 AM occurred twice. Prior to\nversion 0.2.9, this would cause Tron to schedule a daily midnight job to be run\nan hour early on November 7, at 11 PM. For some jobs this doesn't matter, but\nfor jobs that depend on the availability of data for a day, it can cause a\nfailure.\n\nSimilarly, some jobs on March 14, 2011 were scheduled an hour late.\n\nTo avoid this problem, set the :ref:`time_zone` config variable. For example::\n\n time_zone: US/Pacific\n\nIf a job is scheduled at a time that occurs twice, such as 1 AM on \"fall back\",\nit will be run on the *first* occurrence of that time.\n\nIf a job is scheduled at a time that does not exists, such as 2 AM on \"spring\nforward\", it will be run an hour later in the \"new\" time, in this case 3 AM. In\nthe \"old\" time this is 2 AM, so from the perspective of previous jobs, it runs\nat the correct time.\n\nIn general, Tron tries to schedule a job as soon as is correct, and no sooner.\nA job that is schedule for 2:30 AM will not run at 3 AM on \"spring forward\"\nbecause that would be half an hour too soon from a pre-switch perspective (2\nAM).\n\n.. note::\n\n If you experience unexpected scheduler behavior, `file an issue on Tron's\n Github page `_.\n\n.. _job_cleanup_actions:\n\nCleanup Actions\n---------------\n\nCleanup actions run after the job succeeds or fails. They are specified just\nlike regular actions except that there is only one per job and it has no name\nor requirements list.\n\nIf your job creates shared resources that should be destroyed after a run\nregardless of success or failure, such as intermediate files or Amazon Elastic\nMapReduce job flows, you can use cleanup actions to tear them down.\n\nThe command context variable ``cleanup_job_status`` is provided to cleanup\nactions and has a value of ``SUCCESS`` or ``FAILURE`` depending on the job's\nfinal state. For example::\n\n -\n # ...\n cleanup_action:\n command: \"python -m mrjob.tools.emr.job_flow_pool --terminate MY_POOL\"\n\n\nStates\n------\n\nThe following are the possible states for a Job and Job Run.\n\nJob States\n^^^^^^^^^^\n\n**ENABLED**\n A run is scheduled and new runs will continue to be scheduled.\n\n**DISABLED**\n No new runs will be scheduled, and scheduled runs will be cancelled.\n\n**RUNNING**\n Job run currently in progress.\n\nJob Run States\n^^^^^^^^^^^^^^\n\n**SCHE**\n The run is scheduled for a specific time\n\n**RUNN**\n The run is currently running\n\n**SUCC**\n The run completed successfully\n\n**FAIL**\n The run failed\n\n**WAITING**\n The run has actions that are waiting for dependencies\n\n**QUE**\n The run is queued behind another run(s) and will start when said runs finish\n\n**CANC**\n The run was scheduled, but later cancelled.\n\n**UNKWN**\n The run is in an unknown state. This state could indicate a bug in Tron, or\n an exceptional situation with the infrastructure that requires manual inspection.\n Actions for this job may in fact still be running, but Tron cannot reach them.\n\n\nTroubleshooting\n^^^^^^^^^^^^^^^\n**My job doesn't start even though the trigger are emitted?**\n Check that both jobs are in the same tron master. A \"tron master\" refers to a\n cluster; like tron-norcal-devc, tron-nova-prod, etc. Triggers don't work across\n tron masters! You can emit the event manually using command line or API\n\n**S3 consistency issues**\n If your downstream job relies on s3 list to process data you may see it triggered\n before S3 had finished replicating. This was previously masked by log_done\n continuously polling S3 to determine if upstream finished. See STREAMINT-269 for\n details.\n\n.. _shortdate:\n**What does shortdate in triggers mean?**\n There are two concepts of shortdate here.\n\n shortdate in triggered_by: this shortdate is technically the run_date,\n indicating when the tron job runs\n shortdate in command: this shortdate is used by batch jobs to specify which s3\n dir it is writing to or polling.\n" }, { "path": "docs/source/man/tronctl.1", "content": ".TH \"TRONCTL\" \"1\" \"April 24, 2013\" \"0.6\" \"Tron\"\n.SH NAME\ntronctl \\- control Tron jobs and services\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.\\\" Man page generated from reStructeredText.\n.\n.SH SYNOPSYS\n.sp\n\\fBtronctl [\\-\\-server ] [\\-\\-verbose] \\fP\n.SH DESCRIPTION\n.sp\n\\fBtronctl\\fP is the control interface for Tron. \\fBtronctl\\fP allows you to\nenable, disable, start, stop and cancel Tron Jobs and Services.\n.SH OPTIONS\n.INDENT 0.0\n.TP\n.B \\fB\\-\\-server=\\fP\nConfig file containing the address of the server the tron instance is running on\n.TP\n.B \\fB\\-\\-verbose\\fP\nDisplays status messages along the way\n.TP\n.B \\fB\\-\\-run\\-date=\\fP\nFor starting a new job, specifies the run date that should be set. Defaults to today.\n.UNINDENT\n.SH JOB COMMANDS\n.INDENT 0.0\n.TP\n.B disable \nDisables the job. Cancels all scheduled and queued runs. Doesn\\(aqt\nschedule any more.\n.TP\n.B enable \nEnables the job and schedules a new run.\n.TP\n.B start \nCreates a new run of the specified job and runs it immediately.\n.TP\n.B start \nAttempt to start the given job run. A Job run only starts if no\nother instance is running. If the job has already started, it will attempt\nto start any actions in the SCH or QUE state.\n.TP\n.B start \nAttempt to start the action run.\n.TP\n.B restart \nCreates a new job run with the same run time as this job.\n.TP\n.B cancel \nCancels the specified job run or action run.\n.TP\n.B success \nMarks the specified job run or action run as succeeded. This behaves the\nsame as the run actually completing. Dependant actions are run and queued\nruns start.\n.TP\n.B skip \nMarks the specified action run as skipped. This allows dependant actions\nto run, but will not publish any downstream triggers.\n.TP\n.B skip-and-publish \nMarks the specified action run as skipped. This allows dependant actions\nto run and will publish downstream triggers.\n.TP\n.B fail \nMarks the specified job run or action run as failed. This behaves the same\nas the job actually failing.\n.TP\n.B stop \nStop an action run\n.TP\n.B stop \nStop (SIGTERM) a service\n.TP\n.B kill \nForce stop (SIGKILL) a service\n.TP\n.B kill \nForce stop (SIGKILL) an action run\n.TP\n.B move \nRename a job\n.TP\n.B publish \nPulish a action run trigger. This command can be used to trigger an action run in waiting state.\ne.g. \\fBtronctl publish yelp-main.transaction_extract_report.run.shortdate.2019-10-31\\fP\n.TP\n.B discard \nDiscard existing actionrun trigger.\ne.g. \\fBtronctl discard yelp-main.transaction_extract_report.run.shortdate.2019-10-31\\fP\n.TP\n.B version\nPrint tron client and server versions\n\n.UNINDENT\n.SH SERVICE COMMANDS\n.INDENT 0.0\n.TP\n.B start \nStart the service.\n.TP\n.B stop \nStop the service.\n.UNINDENT\n.SH EXAMPLES\n.sp\n.nf\n.ft C\n$ tronctl start job0\nNew Job Run job0.2 created\n\n$ tronctl start job0.3\nJob Run job0.3 now in state RUNN\n\n$ tronctl cancel job0.4\nJob Run job0.4 now in state CANC\n\n$ tronctl fail job0.4\nJob Run job0.4 now in state FAIL\n\n$ tronctl restart job0.4\nJob Run job0.4 now in state RUNN\n\n$ tronctl success job0.5\nJob Run job0.5 now in state SUCC\n.ft P\n.fi\n.SH BUGS\n.sp\nPost bugs to \\fI\\%http://www.github.com/yelp/tron/issues\\fP.\n.SH SEE ALSO\n.sp\n\\fBtrond\\fP (8), \\fBtronfig\\fP (1), \\fBtronview\\fP (1),\n.SH AUTHOR\nYelp, Inc.\n.SH COPYRIGHT\n2011, Yelp, Inc.\n.\\\" Generated by docutils manpage writer.\n.\\\"\n.\n" }, { "path": "docs/source/man/trond.8", "content": ".TH \"TROND\" \"8\" \"April 24, 2013\" \"0.6\" \"Tron\"\n.SH NAME\ntrond \\- trond documentation\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.\\\" Man page generated from reStructeredText.\n.\n.SH SYNOPSYS\n.sp\n\\fBtrond [\\-\\-working\\-dir=] [\\-\\-verbose] [\\-\\-debug]\\fP\n.SH DESCRIPTION\n.sp\n\\fBtrond\\fP is the tron daemon that manages all jobs and services.\n.SH OPTIONS\n.INDENT 0.0\n.TP\n.B \\fB\\-\\-version\\fP\nshow program\\(aqs version number and exit\n.TP\n.B \\fB\\-h, \\-\\-help\\fP\nshow this help message and exit\n.TP\n.B \\fB\\-\\-working\\-dir=WORKING_DIR\\fP\nDirectory where tron\\(aqs state and output is stored (default /var/lib/tron/)\n.TP\n.B \\fB\\-l LOG_CONF, \\-\\-log\\-conf=LOG_CONF\\fP\nLogging configuration file to setup python logger\n.TP\n.B \\fB\\-c CONFIG_FILE, \\-\\-config\\-file=CONFIG_FILE\\fP\nConfiguration file to load (default in working dir)\n.TP\n.B \\fB\\-v, \\-\\-verbose\\fP\nVerbose logging\n.TP\n.B \\fB\\-\\-debug\\fP\nDebug mode, extra error reporting, no daemonizing\n.TP\n.B \\fB\\-\\-nodaemon\\fP\n(DEPRECATED in 0.9.4) Indicates we should not fork and daemonize the process (default False)\n.TP\n.B \\fB\\-\\-lock\\-file=LOCKFILE\\fP\nWhere to store the lock file of the executing process (default /var/run/tron.lock)\n.TP\n.B \\fB\\-P LISTEN_PORT, \\-\\-port=LISTEN_PORT\\fP\nWhat port to listen on, defaults 8089\n.TP\n.B \\fB\\-H LISTEN_HOST, \\-\\-host=LISTEN_HOST\\fP\nWhat host to listen on defaults to localhost\n.UNINDENT\n.SH FILES\n.INDENT 0.0\n.TP\n.B Working directory\nThe directory where state and saved output of processes are stored.\n.TP\n.B Lock file\nEnsures only one daemon runs at a time.\n.TP\n.B Log File\ntrond error log, configured from logging.conf\n.UNINDENT\n.SH SIGNALS\n.INDENT 0.0\n.TP\n.B \\fISIGINT\\fP\nGraceful shutdown. Waits for running jobs to complete.\n.TP\n.B \\fISIGTERM\\fP\nDoes some cleanup before shutting down.\n.TP\n.B \\fISIGHUP\\fP\nReload the configuration file.\n.TP\n.B \\fISIGUSR1\\fP\nWill drop into an ipdb debugging prompt.\n.UNINDENT\n.SH LOGGING\n.sp\nTron uses Python\\(aqs standard logging and by default uses a rotating log file\nhandler that rotates files each day. Logs go to \\fB/var/log/tron/tron.log\\fP.\n.sp\nTo configure logging pass \\-l to trond. You can modify the\ndefault logging.conf by coping it from tron/logging.conf. See\n\\fI\\%http://docs.python.org/howto/logging.html#configuring-logging\\fP\n.SH BUGS\n.sp\ntrond has issues around daylight savings time and may run jobs an hour early\nat the boundary.\n.sp\nPost further bugs to \\fI\\%http://www.github.com/yelp/tron/issues\\fP.\n.SH SEE ALSO\n.sp\n\\fBtronctl\\fP (1), \\fBtronfig\\fP (1), \\fBtronview\\fP (1),\n.SH AUTHOR\nYelp, Inc.\n.SH COPYRIGHT\n2011, Yelp, Inc.\n.\\\" Generated by docutils manpage writer.\n.\\\"\n.\n" }, { "path": "docs/source/man/tronfig.1", "content": ".TH \"TRONFIG\" \"1\" \"April 24, 2013\" \"0.6\" \"Tron\"\n.SH NAME\ntronfig \\- tronfig documentation\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.\\\" Man page generated from reStructeredText.\n.\n.SH SYNOPSYS\n.sp\n\\fBtronfig [\\-\\-server server_name ] [\\-\\-verbose | \\-v] [] [\\-p] [\\-]\\fP\n.SH DESCRIPTION\n.sp\n\\fBtronfig\\fP allows live editing of the Tron configuration. It retrieves\nthe configuration file for local editing, verifies the configuration,\nand sends it back to the tron server. The configuration is applied\nimmediately.\n.SH OPTIONS\n.INDENT 0.0\n.TP\n.B \\fB\\-\\-server \\fP\nThe server the tron instance is running on\n.TP\n.B \\fB\\-\\-verbose\\fP\nDisplays status messages along the way\n.TP\n.B \\fB\\-\\-version\\fP\nDisplays version string\n.TP\n.B \\fB\\-p\\fP\nPrint the configuration\n.TP\n.B \\fBnamespace\\fP\nThe configuration namespace to edit. Defaults to MASTER\n.TP\n.B \\fB\\-\\fP\nRead new config from \\fBstdin\\fP.\n.UNINDENT\n.SH CONFIGURATION\n.sp\nBy default tron will run with a blank configuration file. The config file is\nsaved to \\fB/config/\\fP by default. See the full documentation at\n\\fI\\%http://tron.readthedocs.io/en/latest/config.html\\fP.\n.SH BUGS\n.sp\nPost bugs to \\fI\\%http://www.github.com/yelp/tron/issues\\fP.\n.SH SEE ALSO\n.sp\n\\fBtrond\\fP (8), \\fBtronctl\\fP (1), \\fBtronview\\fP (1),\n.SH AUTHOR\nYelp, Inc.\n.SH COPYRIGHT\n2011, Yelp, Inc.\n.\\\" Generated by docutils manpage writer.\n.\\\"\n.\n" }, { "path": "docs/source/man/tronview.1", "content": ".TH \"TRONVIEW\" \"1\" \"April 24, 2013\" \"0.6\" \"Tron\"\n.SH NAME\ntronview \\- tronview documentation\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.\\\" Man page generated from reStructeredText.\n.\n.SH SYNOPSYS\n.sp\n\\fBtronview [\\-n ] [\\-\\-server ] [\\-\\-verbose] [ | | ]\\fP\n.SH DESCRIPTION\n.sp\n\\fBtronview\\fP displays the status of tron scheduled jobs and services.\n.INDENT 0.0\n.TP\n.B tronview\nShow all configured jobs and services\n.TP\n.B tronview \nShows details for a job or service. Ex:\n.sp\n.nf\n.ft C\n$ tronview my_job\n.ft P\n.fi\n.TP\n.B tronview \nShow details for specific run or instance. Ex:\n.sp\n.nf\n.ft C\n$ tronview my_job.0\n.ft P\n.fi\n.TP\n.B tronview \nShow details for specific action run. Ex:\n.sp\n.nf\n.ft C\n$ tronview my_job.0.my_action\n.ft P\n.fi\n.UNINDENT\n.SH OPTIONS\n.INDENT 0.0\n.TP\n.B \\fB\\-\\-version\\fP\nshow program\\(aqs version number and exit\n.TP\n.B \\fB\\-h, \\-\\-help\\fP\nshow this help message and exit\n.TP\n.B \\fB\\-v, \\-\\-verbose\\fP\nVerbose logging\n.TP\n.B \\fB\\-n NUM_DISPLAYS, \\-\\-numshown=NUM_DISPLAYS\\fP\nThe maximum number of job runs or lines of output to display(0 for show\nall). Does not affect the display of all jobs and the display of actions\nfor given job.\n.TP\n.B \\fB\\-\\-server=SERVER\\fP\nServer URL to connect to\n.TP\n.B \\fB\\-c, \\-\\-color\\fP\nDisplay in color\n.TP\n.B \\fB\\-\\-nocolor\\fP\nDisplay without color\n.TP\n.B \\fB\\-o, \\-\\-stdout\\fP\nSolely displays stdout\n.TP\n.B \\fB\\-e, \\-\\-stderr\\fP\nSolely displays stderr\n.TP\n.B \\fB\\-E\\fP\nlist all emitted triggers\n.TP\n.B \\fB\\-s, \\-\\-save\\fP\nSave server and color options to client config file (~/.tron)\n.UNINDENT\n.SH STATES\n.sp\nFor complete list of states with a diagram of valid transitions see\n\\fI\\%http://packages.python.org/tron/jobs.html#states\\fP and\n\\fI\\%http://packages.python.org/tron/services.html#states\\fP\n.SH BUGS\n.sp\nPost bugs to \\fI\\%http://www.github.com/yelp/tron/issues\\fP.\n.SH SEE ALSO\n.sp\n\\fBtrond\\fP (8), \\fBtronctl\\fP (1), \\fBtronfig\\fP (1),\n.SH AUTHOR\nYelp, Inc.\n.SH COPYRIGHT\n2011, Yelp, Inc.\n.\\\" Generated by docutils manpage writer.\n.\\\"\n.\n" }, { "path": "docs/source/man_tronctl.rst", "content": ".. _tronctl:\n\ntronctl\n=======\n\nSynopsis\n--------\n\n``tronctl [--server ] [--verbose] ``\n\nDescription\n-----------\n\n**tronctl** is the control interface for Tron. :command:`tronctl` allows you to\nenable, disable, start, stop and cancel Tron Jobs and Services.\n\nOptions\n-------\n\n``--server=``\n Config file containing the address of the server the tron instance is running on\n\n``--verbose``\n Displays status messages along the way\n\n``--run-date=``\n For starting a new job, specifies the run date that should be set. Defaults to today.\n\n``--start-date=``\n For backfills, specifies the starting date of the first job of the backfill.\n Note that many jobs operate on the previous day's data.\n\n``--end-date=``\n For backfills, specifies the final date of the backfill. Defaults to today.\n Note that many jobs operate on the previous day's data.\n\nJob Commands\n------------\n\ndisable \n Disables the job. Cancels all scheduled and queued runs. Doesn't\n schedule any more.\n\nenable \n Enables the job and schedules a new run.\n\nstart \n Creates a new run of the specified job and runs it immediately.\n Tron will use the latest version of the code and tron config available for the new run id.\n\nstart \n Attempt to start the given job run. A Job run only starts if no\n other instance is running. If the job has already started, it will attempt\n to start any actions in the SCH or QUE state.\n Tron will use the latest version of the code and tron config available for the new run id.\n\nstart \n Attempt to start the action run.\n Tron will use the latest version of the code and tron config available for the new run id.\n\nrestart \n Creates a new job run with the same run time as this job.\n Tron will use the latest version of the code and tron config available for the new run id.\n\nretry \n Re-run an action within an existing job run.\n Will use the exact same code version and tron config as the previous run.\n\nrerun \n Creates a new job run with the same run time as this job (same as restart).\n Tron will use the latest version of the code and tron config available for the new run id.\n\nbackfill \n Creates a series of tronctl start commands for a sequence of dates.\n --start-date must be provided for a backfill.\n\ncancel \n Cancels the specified job run or action run.\n\nsuccess \n Marks the specified job run or action run as succeeded. This behaves the\n same as the run actually completing. Dependent actions are run and queued\n runs start.\n\nskip \n Marks the specified action run as skipped. This allows dependent actions\n to run, but will not publish any downstream triggers.\n\nskip-and-publish \n Marks the specified action run as skipped. This allows dependent actions\n to run and will publish downstream triggers.\n\nfail \n Marks the specified job run or action run as failed. This behaves the same\n as the job actually failing.\n\nstop \n Stop an action run\n\nkill \n Force stop (SIGKILL) an action run\n\n\nExamples\n--------\n\n::\n\n $ tronctl start job0\n New Job Run job0.2 created\n\n $ tronctl start job0.3\n Job Run job0.3 now in state RUNN\n\n $ tronctl cancel job0.4\n Job Run job0.4 now in state CANC\n\n $ tronctl fail job0.4\n Job Run job0.4 now in state FAIL\n\n $ tronctl restart job0.4\n Job Run job0.4 now in state RUNN\n\n $ tronctl success job0.5\n Job Run job0.5 now in state SUCC\n\n $ tronctl retry MASTER.job.5.action1\n Retrying ActionRun: MASTER.job.5.action1\n\nBugs\n----\n\nPost bugs to http://www.github.com/yelp/tron/issues.\n\nSee Also\n--------\n\n**trond** (8), **tronfig** (1), **tronview** (1),\n" }, { "path": "docs/source/man_trond.rst", "content": ".. _trond:\n\ntrond\n=====\n\nSynopsis\n--------\n\n``trond [--working-dir=] [--verbose] [--debug]``\n\nDescription\n-----------\n\n**trond** is the tron daemon that manages all jobs.\n\nOptions\n-------\n\n``--version``\n show program's version number and exit\n\n``-h, --help``\n show this help message and exit\n\n``--working-dir=WORKING_DIR``\n Directory where tron's state and output is stored (default /var/lib/tron/)\n\n``-l LOG_CONF, --log-conf=LOG_CONF``\n Logging configuration file to setup python logger\n\n``-c CONFIG_FILE, --config-file=CONFIG_FILE``\n Configuration file to load (default in working dir)\n\n``-v, --verbose``\n Verbose logging\n\n``--debug``\n Debug mode, extra error reporting, no daemonizing\n\n``--nodaemon``\n [DEPRECATED in 0.9.4] Indicates we should not fork and daemonize the process (default False)\n\n``--lock-file=LOCKFILE``\n Where to store the lock file of the executing process (default /var/run/tron.lock)\n\n``-P LISTEN_PORT, --port=LISTEN_PORT``\n What port to listen on, defaults 8089\n\n``-H LISTEN_HOST, --host=LISTEN_HOST``\n What host to listen on defaults to localhost\n\nFiles\n-----\n\nWorking directory\n The directory where state and saved output of processes are stored.\n\nLock file\n Ensures only one daemon runs at a time.\n\nLog File\n trond error log, configured from logging.conf\n\n\nSignals\n-------\n\n`SIGINT`\n Graceful shutdown. Waits for running jobs to complete.\n\n`SIGTERM`\n Does some cleanup before shutting down.\n\n`SIGHUP`\n Reload the configuration file.\n\n`SIGUSR1`\n Will drop into an ipdb debugging prompt.\n\nLogging\n-------\n\nTron uses Python's standard logging and by default uses a rotating log file\nhandler that rotates files each day. Logs go to ``/var/log/tron/tron.log``.\n\nTo configure logging pass -l to trond. You can modify the\ndefault logging.conf by coping it from tron/logging.conf. See\nhttp://docs.python.org/howto/logging.html#configuring-logging\n\n\nBugs\n----\n\ntrond has issues around daylight savings time and may run jobs an hour early\nat the boundary.\n\nPost further bugs to http://www.github.com/yelp/tron/issues.\n\nSee Also\n--------\n\n**tronctl** (1), **tronfig** (1), **tronview** (1),\n" }, { "path": "docs/source/man_tronfig.rst", "content": ".. _tronfig:\n\ntronfig\n=======\n\nSynopsis\n--------\n\n``tronfig [--server server_name ] [--verbose | -v] [] [-p] [-]``\n\nDescription\n-----------\n\n**tronfig** allows live editing of the Tron configuration. It retrieves\nthe configuration file for local editing, verifies the configuration,\nand sends it back to the tron server. The configuration is applied\nimmediately.\n\nOptions\n-------\n\n``--server ``\n The server the tron instance is running on\n\n``--verbose``\n Displays status messages along the way\n\n``--version``\n Displays version string\n\n``-p``\n Print the configuration\n\n``namespace``\n The configuration namespace to edit. Defaults to MASTER\n\n``-``\n Read new config from ``stdin``.\n\nConfiguration\n-------------\n\nBy default tron will run with a blank configuration file. The config file is\nsaved to ``/config/`` by default. See the full documentation at\nhttp://tron.readthedocs.io/en/latest/config.html.\n\n\nBugs\n----\n\nPost bugs to http://www.github.com/yelp/tron/issues.\n\nSee Also\n--------\n\n**trond** (8), **tronctl** (1), **tronview** (1),\n" }, { "path": "docs/source/man_tronview.rst", "content": ".. _tronview:\n\ntronview\n========\n\nSynopsis\n--------\n\n``tronview [-n ] [--server ] [--verbose] [ | | ]``\n\nDescription\n-----------\n\n**tronview** displays the status of tron scheduled jobs.\n\ntronview\n Show all configured jobs\n\ntronview \n Shows details for a job. Ex::\n\n $ tronview my_job\n\ntronview \n Show details for specific run or instance. Ex::\n\n $ tronview my_job.0\n\ntronview \n Show details for specific action run. Ex::\n\n $ tronview my_job.0.my_action\n\nOptions\n-------\n\n``--version``\n show program's version number and exit\n\n``-h, --help``\n show this help message and exit\n\n``-v, --verbose``\n Verbose logging\n\n``-n NUM_DISPLAYS, --numshown=NUM_DISPLAYS``\n The maximum number of job runs or lines of output to display(0 for show\n all). Does not affect the display of all jobs and the display of actions\n for given job.\n\n``--server=SERVER``\n Server URL to connect to\n\n``-c, --color``\n Display in color\n\n``--nocolor``\n Display without color\n\n``-o, --stdout``\n Solely displays stdout\n\n``-e, --stderr``\n Solely displays stderr\n\n``-s, --save``\n Save server and color options to client config file (~/.tron)\n\n\nStates\n----------\nFor complete list of states with a diagram of valid transitions see\nhttp://packages.python.org/tron/jobs.html#states\n\n\nBugs\n----\n\nPost bugs to http://www.github.com/yelp/tron/issues.\n\nSee Also\n--------\n\n**trond** (8), **tronctl** (1), **tronfig** (1),\n" }, { "path": "docs/source/overview.rst", "content": "Overview\n========\n\nBatch process scheduling on a single UNIX machines has\nhistorically been managed by :command:`cron` and its derivatives. But if you\nhave many batches, complex dependencies between batches, or many machines,\nmaintaining config files across them may be difficult. Tron solves this\nproblem by centralizing the configuration and scheduling of jobs to a single daemon.\n\nThe Tron system is split into four commands:\n\n:ref:`trond`\n Daemon responsible for scheduling, running, and saving state. Provides an\n HTTP interface to tools.\n\n:ref:`tronview`\n View job state and output.\n\n:ref:`tronctl`\n Start, stop, enable, disable, and otherwise control jobs.\n\n:ref:`tronfig`\n Change Tron's configuration while the daemon is still running.\n\nThe config file uses YAML syntax, and is further described in :doc:`config`.\n\nNodes, Jobs and Actions\n-----------------------\n\nTron's orders consist of *jobs*. :doc:`Jobs ` contain\n:ref:`actions ` which may depend on other actions in the same job\nand run on a schedule.\n\n:command:`trond` is given access (via public key SSH) to one or more *nodes* on\nwhich to run jobs. For example, this configuration has two nodes,\neach of which is responsible for a single job::\n\n nodes:\n hostname: 'localhost'\n - name: node1\n hostname: 'batch1'\n - name: node2\n hostname: 'batch2'\n\n jobs:\n \"job0\":\n node: node1\n schedule: \"cron * * * * *\"\n actions:\n \"batch1action\":\n command: \"sleep 3; echo asdfasdf\"\n \"job1\":\n node: node2\n schedule: \"cron * * * * *\"\n actions:\n \"batch2action\":\n command: \"cat big.txt; sleep 10\"\n\n\nHow the nodes are set up and assigned to jobs is entirely up to you. They may\nhave different operating systems, access to different databases, different\nprivileges for the Tron user, etc.\n\nSee also:\n\n* :doc:`jobs`\n* :doc:`config`\n\n.. _overview_pools:\n\nNode Pools\n----------\n\nNodes can be grouped into *pools*. To continue the previous example::\n\n node_pools:\n - name:pool\n nodes: [node1, node2]\n\n jobs:\n # ...\n \"job2\":\n node: pool\n schedule: \"cron * * * * *\"\n actions:\n \"pool_action\":\n command: \"ls /; sleep 1\"\n cleanup_action:\n command: \"echo 'all done'\"\n\n``job2``'s action will be run on a random node from ``pool`` every 5 seconds.\nWhen ``pool_action`` is complete, ``cleanup_action`` will run on the same node.\n\nFor more information, see :doc:`jobs`.\n\nCaveats\n-------\n\nWhile Tron solves many scheduling-related problems, there are a few things to\nwatch out for.\n\n**Tron keeps an SSH connection open for the entire lifespan of a process.**\nThis means that to upgrade :command:`trond`, you have to either wait until no\njobs are running, or accept an inconsistent state. This limitation is being\nworked on, and should be improved in later releases.\n\n**Tron is under active development.** This means that some things will change.\nWhenever possible these changes will be backwards compatible, but in some\ncases there may be non-backwards compatible changes.\n\n**Tron does not support unicode.** Tron is built using `twisted `_\nwhich does not support unicode.\n" }, { "path": "docs/source/sample_config.yaml", "content": "# optional and settable from the command line\nworking_dir: './working'\n\n# optional\nssh_options:\n agent: true # default False\n identities: # default []\n - \"/home/batch/.ssh/id_dsa-nopasswd\"\n\ncommand_context:\n PYTHON: /usr/bin/python\n TMPDIR: /tmp\n\n# required\nnodes:\n - name: node1\n hostname: 'batch1'\n username: 'tronuser'\n - name: node2\n hostname: 'batch2'\n username: 'tronuser'\n\nnode_pools:\n - name: pool\n nodes: [node1, node2]\n\njobs:\n \"job0\":\n node: pool\n all_nodes: True\n schedule:\n start_time: 04:00:00\n queueing: False\n actions:\n \"verify_logs_present\":\n command: >\n ls /var/log/app/log_{shortdate-1}.txt\n \"convert_logs\":\n command: >\n convert_logs /var/log/app/log_{shortdate-1}.txt \\\n /var/log/app_converted/log_{shortdate-1}.txt\n requires: [verify_logs_present]\n # this will run when the job succeeds or fails\n cleanup_action:\n command: \"rm /{TMPDIR}/random_temp_file\"\n\n \"job1\":\n node: node\n schedule: \"every monday at 09:00\"\n queueing: False\n actions:\n \"actionAlone\":\n command: \"cat big.txt; sleep 10\"\n" }, { "path": "docs/source/tools.rst", "content": "Man Pages\n=========\n\n.. toctree::\n :maxdepth: 2\n\n man_tronctl.rst\n man_trond.rst\n man_tronfig.rst\n man_tronview.rst\n" }, { "path": "docs/source/tron.yaml", "content": "ssh_options:\n agent: true\n\nnodes:\n - name: node0\n hostname: 'localhost'\n\njobs:\n \"uptime_job\":\n node: node0\n schedule: \"cron */10 * * * *\"\n actions:\n \"uptimer\":\n command: \"uptime\"\n" }, { "path": "docs/source/tronweb.rst", "content": ".. _tronweb:\n\ntronweb\n========\n\ntronweb is the web-based UI for tron.\n\nSee http://localhost:8089/web/\n" }, { "path": "docs/source/tutorial.rst", "content": "Tutorial\n========\n\nTo install Tron you will need:\n\n* A copy of the most recent Tron release from either\n `github `_ or `pypi `_\n (see :ref:`installing_tron`).\n* A server on which to run :command:`trond`.\n* One or more batch boxes which will run the Jobs.\n* An SSH key and a user that will allow the tron daemon to login to all of the\n batch machines without a password prompt.\n\n.. _installing_tron:\n\nInstalling Tron\n---------------\n\nThe easiest way to install Tron is from PyPI::\n\n $ sudo pip install tron\n\nYou can also get a copy of the current development release from\n`github `_. See `setup.py` in the source package\nfor a full list of required packages.\n\nIf you are interested in working on Tron development see :ref:`developing`\nfor additional requirements and setting up a dev environment.\n\n\nRunning Tron\n-------------\n\nTron runs as a single daemon, :command:`trond`.\n\nOn your management node, run::\n\n $ sudo -u trond\n\nThe chosen user will need SSH access to all your worker nodes, as well as\npermission to write to the working directory, log file, and lock file\n(see ``trond --help`` for defaults). You can change these directories using\ncommand line options. Also see :ref:`config_logging` on how to change the\ndefault logging settings.\n\n\nOnce :command:`trond` is running, you can view its status using :command:`tronview`\n(by default tronview will connect to localhost, use ``--server=: -s``\nto specify a different server, and have that setting saved in ``~/.tron``)::\n\n $ tronview\n\n Jobs:\n No jobs\n\nConfiguring Tron\n----------------\n\nThere are a few options on how to configure tron, but the most straightforward\nis through tronfig::\n\n $ tronfig\n\nThis will open your configured :envvar:`$EDITOR` with the current configuration\nfile. Edit your file to be something like this::\n\n ssh_options:\n agent: true\n\n nodes:\n - name: local\n hostname: 'localhost'\n\n jobs:\n \"getting_node_info\":\n node: local\n schedule: \"cron */10 * * * *\"\n actions:\n \"uname\":\n command: \"uname -a\"\n \"cpu_info\":\n command: \"cat /proc/cpuinfo\"\n requires: [uname]\n\nAfter you exit your editor, the configuration will be validated and uploaded to `trond`.\n\nNow if you run :command:`tronview` again, you'll see ``getting_node_info`` as a\nconfigured job. Note that it is configured to run 10 minutes from now. This\nshould give you time to examine the job to ensure you really want to run it.\n\n::\n\n Jobs:\n Name State Scheduler Last Success\n getting_node_info ENABLED INTERVAL:0:10:00 None\n\nYou can quickly disable a job by using :command:`tronctl`::\n\n $ tronctl disable getting_node_info\n Job getting_node_info is disabled\n\nThis will stop scheduled jobs and prevent anymore from being scheduled. You are\nnow in manual control. To manually execute a job immediately, do this::\n\n $ tronctl start getting_node_info\n New job getting_node_info.1 created\n\nYou can monitor this job run by using :command:`tronview`::\n\n $ tronview getting_node_info.1\n Job Run: getting_node_info.1\n State: SUCC\n Node: localhost\n\n Action ID & Command State Start Time End Time Duration\n .uname SUCC 2011-02-28 16:57:48 2011-02-28 16:57:48 0:00:00\n .cpu_info SUCC 2011-02-28 16:57:48 2011-02-28 16:57:48 0:00:00\n\n $ tronview getting_node_info.1.uname\n Action Run: getting_node_info.1.uname\n State: SUCC\n Node: localhost\n\n uname -a\n\n Requirements:\n\n Stdout:\n Linux dev05 2.6.24-24-server #1 SMP Wed Apr 15 15:41:09 UTC 2009 x86_64 GNU/Linux\n Stderr:\n\nTron also provides a simple, optional web UI that can be used to get tronview data in a browser. See :doc:`tronweb` for setup\ninstructions.\n\nThat's it for the basics. You might want to look at :doc:`overview` for a more\ncomprehensive description of how Tron works.\n" }, { "path": "docs/source/whats-new.rst", "content": "What's New\n==========\n\nSee the `CHANGELOG `_.\n" }, { "path": "itest.sh", "content": "#!/bin/bash\n\nset -euxo pipefail\n\nexport DEBIAN_FRONTEND=noninteractive\n\napt-get update\napt-get install -y software-properties-common gdebi-core curl\nadd-apt-repository -y ppa:deadsnakes/ppa\napt-get update\n\ngdebi --non-interactive /work/dist/*.deb\n\n# TODO: change default MASTER config to not require ssh agent\napt-get install -y ssh\nservice ssh start\neval $(ssh-agent)\n\ntrond --help\ntronfig --help\n\n\n/opt/venvs/tron/bin/python - </dev/null; then\n break\n fi\n if [ \"$i\" == \"5\" ]; then\n echo \"Failed to start\"\n kill -9 $TRON_PID\n exit 1\n fi\n sleep 1\ndone\nkill -0 $TRON_PID\n\ncurl localhost:8089/api/status | grep -qi alive\n\ntronfig -p MASTER\ntronfig -n MASTER /work/testfiles/MASTER.yaml\ntronfig /work/testfiles/MASTER.yaml\ncat /work/testfiles/MASTER.yaml | tronfig -n MASTER -\n\nif test -L /opt/venvs/tron/lib/python3.10/encodings/punycode.py; then\n echo \"Whoa, the tron package shouldn't have an encoding symlink!\"\n echo \"Check out https://github.com/spotify/dh-virtualenv/issues/272\"\n exit 1\nfi\n\nkill -SIGTERM $TRON_PID\nwait $TRON_PID || true\n" }, { "path": "mypy.ini", "content": "[mypy]\npython_version = 3.10\n# TODO: we'd like to be as strict as we are internally, but we need to fully type Tron first\n# disallow_any_generics = true\ndisallow_incomplete_defs = True\n# disallow_untyped_calls = true\ndisallow_untyped_decorators = True\n# disallow_untyped_defs = true\n\nshow_column_numbers = True\nshow_error_codes = True\nshow_error_context = True\n\nwarn_incomplete_stub = True\nwarn_redundant_casts = True\nwarn_return_any = True\nwarn_unreachable = True\nwarn_unused_ignores = True\n\nexclude = .tox/\n\n[mypy-clusterman_metrics.*]\nignore_missing_imports = True\n\n[mypy-twisted.internet.*]\nignore_missing_imports = True\n" }, { "path": "osx-bdb.sh", "content": "#!/bin/bash\n\nexport BERKELEYDB_DIR=$(brew --prefix berkeley-db)\nexport YES_I_HAVE_THE_RIGHT_TO_USE_THIS_BERKELEY_DB_VERSION=1\n" }, { "path": "package.json", "content": "{\n \"private\": true,\n \"homepage\": \"./\",\n \"dependencies\": {},\n \"browserslist\": {\n \"production\": [\n \">0.2%\",\n \"not dead\",\n \"not op_mini all\"\n ],\n \"development\": [\n \"last 1 chrome version\",\n \"last 1 firefox version\",\n \"last 1 safari version\"\n ]\n },\n \"devDependencies\": {\n \"eslint\": \"^6.6.0\",\n \"eslint-config-airbnb\": \"^18.2.0\",\n \"eslint-plugin-import\": \"^2.22.0\",\n \"eslint-plugin-jsx-a11y\": \"^6.3.1\",\n \"eslint-plugin-react\": \"^7.20.6\",\n \"eslint-plugin-react-hooks\": \"^4.1.0\"\n },\n \"resolutions\": {\n \"axe-core\": \"4.7.0\"\n },\n \"engines\": {\n \"node-version-shim\": \"10.x\",\n \"node\": \">=10\"\n }\n}\n" }, { "path": "pyproject.toml", "content": "[tool.black]\nline-length = 120\ntarget_version = ['py310']\n" }, { "path": "requirements-dev-minimal.txt", "content": "asynctest\nbotocore-stubs\ndebugpy\nflake8\nmoto\nmypy\npre-commit\npytest\npytest-asyncio\nrequirements-tools\ntypes-boto3\ntypes-cachetools\ntypes-psutil\ntypes-pytz\ntypes-PyYAML\ntypes-requests<2.31.0.7 # newer types-requests requires urllib3>=2\ntypes-setuptools\n" }, { "path": "requirements-dev.txt", "content": "asynctest==0.12.0\nboto3-stubs==1.35.63\nbotocore-stubs==1.38.19\ncfgv==2.0.1\ndebugpy==1.8.1\ndistlib==0.3.6\nfilelock==3.4.1\nflake8==5.0.4\nidentify==2.5.5\niniconfig==1.1.1\nmccabe==0.7.0\nmoto==4.1.0\nmypy==1.9.0\nmypy-extensions==1.0.0\nnodeenv==1.8.0\npackaging==19.2\nplatformdirs==2.5.2\npluggy==0.13.0\npre-commit==2.20.0\npy==1.10.0\npycodestyle==2.9.0\npyflakes==2.5.0\npyparsing==2.4.2\npytest==7.0.1\npytest-asyncio==0.14.0\nrequirements-tools==2.1.0\nresponses==0.13.0\ntoml==0.10.2\ntomli==2.0.1\ntypes-awscrt==0.27.2\ntypes-boto3==1.0.2\ntypes-cachetools==5.5.0.20240820\ntypes-psutil==6.1.0.20241221\ntypes-pytz==2024.2.0.20240913\ntypes-PyYAML==6.0.12\ntypes-requests==2.31.0.5\ntypes-s3transfer==0.13.0\ntypes-setuptools==75.8.0.20250110\ntypes-urllib3==1.26.25.14\nvirtualenv==20.17.1\nxmltodict==0.12.0\n" }, { "path": "requirements-docs.txt", "content": "Jinja2==3.1.2\nmarkupsafe==2.1.1\nmock==3.0.5\nPygments==2.13.0\nSphinx==6.1.3\nSphinx-PyPI-upload==0.2.1\n" }, { "path": "requirements-minimal.txt", "content": "addict # not sure why check-requirements is not picking this up from task_processing[mesos_executor]\nargcomplete\nboto3\nbsddb3\ncryptography\ndataclasses\necdsa>=0.13.3\nhttp-parser # not sure why check-requirements is not picking this up from task_processing[mesos_executor]\nhumanize\nipdb\nipython\nJinja2>=3.1.2\nlockfile\nprometheus-client\npsutil\npy-bcrypt\npyasn1\npyformance\npymesos # not sure why check-requirements is not picking this up from task_processing[mesos_executor]\npyopenssl # vault-tools dependency, but mypy is picking up some code (that we don't use) that imports pyopenssl in Twisted here we are. we could add Twisted[tls], but while hacky - this feels a little more explicit\npysensu-yelp\nPyStaticConfiguration\npytimeparse\npytz\nPyYAML>=5.1\nrequests\ntask_processing[mesos_executor,k8s]>=1.2.0\nTwisted>=19.7.0\nurllib3>=1.24.2\nWerkzeug>=0.15.3\n" }, { "path": "requirements.txt", "content": "addict==2.2.1\nargcomplete==1.9.5\nasttokens==2.2.1\nattrs==19.3.0\nAutomat==20.2.0\nbackcall==0.1.0\nboto3==1.34.80\nbotocore==1.34.80\nbsddb3==6.2.7\ncachetools==4.2.1\ncertifi==2022.12.7\ncffi==1.15.0\ncharset-normalizer==2.0.12\nconstantly==15.1.0\ncryptography==41.0.5\ndataclasses==0.6\ndecorator==4.4.0\necdsa==0.13.3\nexecuting==1.2.0\ngoogle-auth==1.23.0\nhttp-parser==0.9.0\nhumanize==4.10.0\nhyperlink==19.0.0\nidna==2.8\nincremental==22.10.0\nipdb==0.13.2\nipython==8.10.0\nipython-genutils==0.2.0\njedi==0.16.0\nJinja2==3.1.2\njmespath==0.9.4\nkubernetes==26.1.0\nlockfile==0.12.2\nMarkupSafe==2.1.1\nmatplotlib-inline==0.1.3\noauthlib==3.1.0\nparso==0.7.0\npexpect==4.7.0\npickleshare==0.7.5\nprometheus-client==0.21.1\nprompt-toolkit==3.0.38\npsutil==5.6.6\nptyprocess==0.6.0\npure-eval==0.2.2\npy-bcrypt==0.4\npyasn1==0.4.7\npyasn1-modules==0.2.8\npycparser==2.19\npyformance==0.4\nPygments==2.13.0\npymesos==0.3.9\npyopenssl==24.2.1\npyrsistent==0.15.4\npysensu-yelp==1.0.3\nPyStaticConfiguration==0.11.1\npython-dateutil==2.8.1\npytimeparse==1.1.8\npytz==2019.3\nPyYAML==6.0.1\nrequests==2.27.1\nrequests-oauthlib==1.2.0\nrsa==4.9\ns3transfer==0.10.1\nsetuptools==65.5.1\nsix==1.15.0\nstack-data==0.6.2\ntask-processing==1.3.5\ntraitlets==5.0.0\nTwisted==22.10.0\ntyping-extensions==4.5.0\nurllib3==1.25.10\nwcwidth==0.1.7\nwebsocket-client==0.56.0\nWerkzeug==2.2.3\nzope.interface==7.2\n" }, { "path": "setup.cfg", "content": "[build_docs]\nsource-dir = docs/\nbuild-dir = docs/_build\nall_files = 1\n\n[upload_docs]\nupload_dir = docs/_build/html\n" }, { "path": "setup.py", "content": "try:\n from setuptools import setup, find_packages\n\n assert setup\nexcept ImportError:\n from distutils.core import setup\n\nimport glob\nimport tron\n\nsetup(\n name=\"tron\",\n version=tron.__version__,\n provides=[\"tron\"],\n author=\"Yelp\",\n author_email=\"yelplabs@yelp.com\",\n url=\"http://github.com/Yelp/Tron\",\n description=\"Job scheduling and monitoring system\",\n classifiers=[\n \"Programming Language :: Python\",\n \"Programming Language :: Python :: 3.10\",\n \"Operating System :: OS Independent\",\n \"License :: OSI Approved :: Apache Software License\",\n \"Topic :: System :: Monitoring\",\n \"Topic :: System :: Systems Administration\",\n \"Intended Audience :: Developers\",\n \"Intended Audience :: System Administrators\",\n \"Development Status :: 4 - Beta\",\n ],\n packages=find_packages(\n exclude=[\"tests.*\", \"tests\", \"example-cluster\"],\n )\n + [\"tronweb\"],\n scripts=glob.glob(\"bin/*\") + glob.glob(\"tron/bin/*.py\"),\n include_package_data=True,\n long_description=\"\"\"\nTron is a centralized system for managing periodic batch processes across a\ncluster. If you find cron or fcron to be insufficient for managing complex work\nflows across multiple computers, Tron might be for you.\n\nFor more information, look at the\n`tutorial `_ or the\n`full documentation `_.\n\"\"\",\n)\n" }, { "path": "testfiles/MASTER.yaml", "content": "eventbus_enabled: true\n\nstate_persistence:\n name: \"/nail/tron/tron_state\"\n store_type: \"shelve\"\n buffer_size: 10\n\nssh_options:\n agent: False\n identities:\n - /work/example-cluster/insecure_key\n\naction_runner:\n runner_type: \"subprocess\"\n remote_status_path: \"/tmp/tron\"\n remote_exec_path: \"/work/tron/bin/\"\n\nnodes:\n - hostname: localhost\n username: root\n\ntime_zone: US/Eastern\n\njobs:\n one:\n node: localhost\n schedule: \"cron */5 * * * *\"\n actions:\n one:\n command: exit 1\n retries: 3\n retries_delay: 1m\n trigger_downstreams: {ymdhm: \"{ymdhm}\"}\n two:\n node: localhost\n schedule: \"cron */5 * * * *\"\n actions:\n two:\n command: sleep 10 && date\n triggered_by: [\"MASTER.one.one.ymdhm.{ymdhm}\"]\n trigger_timeout: 30s\n" }, { "path": "testifycompat/__init__.py", "content": "from unittest import TestCase # noqa: F401\n\nfrom testifycompat.assertions import * # noqa: F401, F403\nfrom testifycompat.fixtures import * # noqa: F401, F403\n\n\nversion = \"0.1.2\"\n\n\ndef run():\n raise AssertionError(\n \"Oops, you tried to use testifycompat.run(). This function doesn't \"\n \"do anything, it only exists as backwards compatibility with testify. \"\n \"You should remove it from your code.\",\n )\n" }, { "path": "testifycompat/assertions.py", "content": "\"\"\"Compatiblity functions for py.test to migrate code from testify.\n\nThis is not a complete list, but should hopefully cover most of the common\nassertions.\n\nThese assertions should **not** be used in new code, and are only for migrating\nold tests.\n\"\"\"\nimport pytest\n\n\ndef assert_equal(left, right, *args):\n assert left == right\n\n\nassert_sets_equal = assert_dicts_equal = assert_datetimes_equal = assert_equal\nassert_equals = assert_equal\n\n\ndef assert_true(val):\n assert val\n\n\ndef assert_false(val):\n assert not val\n\n\ndef assert_raises_and_contains(exc, text, func, *args, **kwargs):\n with pytest.raises(exc) as excinfo:\n func(*args, **kwargs)\n\n text = text if isinstance(text, list) else [text]\n for item in text:\n assert item in str(excinfo.exconly())\n\n\ndef assert_raises(exc, func=None, *args, **kwargs):\n if func is None:\n return pytest.raises(exc)\n\n with pytest.raises(exc):\n func(*args, **kwargs)\n\n\ndef assert_in(item, container):\n assert item in container\n\n\ndef assert_not_in(item, container):\n assert item not in container\n\n\ndef assert_is(left, right):\n assert left is right\n\n\ndef assert_is_not(left, right):\n assert left is not right\n\n\ndef assert_not_equal(left, right):\n assert left != right\n\n\ndef assert_lt(left, right):\n assert left < right\n\n\ndef assert_lte(left, right):\n assert left <= right\n\n\ndef assert_gt(left, right):\n assert left > right\n\n\ndef assert_gte(left, right):\n assert left >= right\n\n\ndef assert_in_range(val, start, end):\n assert start < val < end\n\n\ndef assert_between(val, start, end):\n assert start <= val <= end\n\n\ndef assert_all_in(left, right):\n \"\"\"Assert that everything in `left` is also in `right`\n Note: This is different than `assert_subset()` because python sets use\n `__hash__()` for comparision whereas `in` uses `__eq__()`.\n \"\"\"\n for item in left:\n assert item in right\n\n\ndef assert_starts_with(val, prefix):\n assert val.startswith(prefix)\n\n\ndef assert_not_reached():\n assert False\n\n\ndef assert_empty(iterable):\n assert len(list(iterable)) == 0\n\n\ndef assert_not_empty(iterable):\n assert len(list(iterable)) > 0\n\n\ndef assert_length(sequence, expected):\n assert len(list(sequence)) == expected\n\n\ndef assert_sorted_equal(left, right):\n assert sorted(left) == sorted(right)\n\n\ndef assert_isinstance(object_, type_):\n assert isinstance(object_, type_)\n" }, { "path": "testifycompat/bin/__init__.py", "content": "" }, { "path": "testifycompat/bin/migrate.py", "content": "#!/usr/bin/env python\n\"\"\"\n\n.. warning::\n\n This script is still very experimental. Use at your own risk. It will\n be replaced over time with lib2to3 fixers.\n\n\nUsage:\n\n ``python -m testifycompat.bin.migrate ``\n\nExample:\n\n ``find tests -name *.py | xargs python migrate.py``\n\n\n\"\"\"\nimport functools\nimport re\nimport sys\n\n\ndef replace(pattern, repl):\n return functools.partial(re.sub, pattern, repl)\n\n\nreplaces = [\n # Replace imports\n replace(r\"^from testify import \", \"from testifycompat import \"),\n replace(r\"^from testify.assertions import \", \"from testifycompat import \"),\n replace(r\"^import testify as T\", \"import testifycompat as T\"),\n # Replace test classes\n replace(\n r\"^class (?:Test)?(\\w+)(?:Test|TestCase)\\((?:T\\.)?TestCase\\):$\",\n \"class Test\\\\1(object):\",\n ),\n replace(\n r\"^class (?:Test)?(\\w+)(?:Test|TestCase)(\\(\\w+TestCase\\)):$\",\n \"class Test\\\\1\\\\2:\",\n ),\n # Replace some old assertions\n replace(r\"self.assert_\\((.*)\\)\", \"assert \\\\1\"),\n]\n\n\ndef run_replacement(contents):\n for line in contents:\n for replacement in replaces:\n line = replacement(line)\n yield line\n\n\ndef strip_if_main_run(contents):\n if len(contents) < 2:\n return contents\n if \"run()\" in contents[-1] and \"if __name__ == \" in contents[-2]:\n return contents[:-2]\n return contents\n\n\ndef run_migration_on_file(filename):\n with open(filename) as fh:\n lines = fh.read().split(\"\\n\")\n\n lines = list(run_replacement(lines))\n lines = strip_if_main_run(lines)\n\n with open(filename, \"w\") as fh:\n fh.write(\"\\n\".join(lines))\n\n\nif __name__ == \"__main__\":\n for filename in sys.argv[1:]:\n run_migration_on_file(filename)\n" }, { "path": "testifycompat/fixtures.py", "content": "\"\"\"\nCompatibility fixtures for migrating code from testify to py.test\n\n.. note::\n\n ``class_`` fixtures must be applied to @classmethods. py.test will not run\n a class_* fixture that is not attached to a class-level method, so your\n tests will probably fail.\n\"\"\"\nimport pytest\n\n\ndef setup(func):\n return pytest.fixture(autouse=True)(func)\n\n\ndef setup_teardown(func):\n return pytest.yield_fixture(autouse=True)(func)\n\n\ndef teardown(func):\n def teardown_(*args, **kwargs):\n yield\n func(*args, **kwargs)\n\n return pytest.yield_fixture(autouse=True)(teardown_)\n\n\ndef class_setup(func):\n return pytest.fixture(autouse=True, scope=\"class\")(func)\n\n\ndef class_setup_teardown(func):\n return pytest.yield_fixture(autouse=True, scope=\"class\")(func)\n\n\ndef class_teardown(func):\n def teardown_(*args, **kwargs):\n yield\n func(*args, **kwargs)\n\n return pytest.yield_fixture(autouse=True, scope=\"class\")(teardown_)\n\n\ndef suite(name, reason=None):\n \"\"\"Translate a :func:`testify.suite` decorator into the appropriate\n :mod:`pytest.mark` call. For the disabled suite this results in a\n skipped test. For other suites it will return a `pytest.mark.`\n decorator.\n \"\"\"\n if name == \"disabled\":\n return pytest.mark.skipif(True, reason=reason)\n\n return getattr(pytest.mark, name)\n" }, { "path": "tests/__init__.py", "content": "from twisted.python import log\n\nobserver = log.PythonLoggingObserver()\nobserver.start()\n" }, { "path": "tests/actioncommand_test.py", "content": "import shlex\nfrom unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_not_equal\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tests.testingutils import autospec_method\nfrom tron import actioncommand\nfrom tron.actioncommand import ActionCommand\nfrom tron.config import schema\nfrom tron.serialize import filehandler\n\n\nclass TestActionCommand(TestCase):\n @setup\n def setup_command(self):\n self.serializer = mock.create_autospec(filehandler.FileHandleManager)\n self.serializer.open.return_value = filehandler.NullFileHandle\n self.ac = ActionCommand(\"action.1.do\", \"do\", self.serializer)\n\n def test_init(self):\n assert_equal(self.ac.state, ActionCommand.PENDING)\n\n def test_init_no_serializer(self):\n ac = ActionCommand(\"action.1.do\", \"do\")\n ac.write_stdout(\"something\")\n ac.write_stderr(\"else\")\n assert_equal(ac.stdout, filehandler.NullFileHandle)\n ac.done()\n\n def test_started(self):\n assert self.ac.started()\n assert self.ac.start_time is not None\n assert_equal(self.ac.state, ActionCommand.RUNNING)\n\n def test_started_already_started(self):\n self.ac.started()\n assert not self.ac.started()\n\n def test_exited(self):\n self.ac.started()\n assert self.ac.exited(123)\n assert_equal(self.ac.exit_status, 123)\n assert self.ac.end_time is not None\n\n def test_exited_from_pending(self):\n assert self.ac.exited(123)\n assert_equal(self.ac.state, ActionCommand.FAILSTART)\n\n def test_exited_bad_state(self):\n self.ac.started()\n self.ac.exited(123)\n assert not self.ac.exited(1)\n\n def test_write_stderr_no_fh(self):\n message = \"this is the message\"\n # Test without a stderr\n self.ac.write_stderr(message)\n\n def test_write_stderr(self):\n message = \"this is the message\"\n serializer = mock.create_autospec(filehandler.FileHandleManager)\n fh = serializer.open.return_value = mock.create_autospec(\n filehandler.FileHandleWrapper,\n )\n ac = ActionCommand(\"action.1.do\", \"do\", serializer)\n\n ac.write_stderr(message)\n fh.write.assert_called_with(message)\n\n def test_done(self):\n self.ac.started()\n self.ac.exited(123)\n assert self.ac.done()\n\n def test_done_bad_state(self):\n assert not self.ac.done()\n\n def test_handle_errback(self):\n message = \"something went wrong\"\n self.ac.handle_errback(message)\n assert_equal(self.ac.state, ActionCommand.FAILSTART)\n assert self.ac.end_time\n\n def test_is_unknown(self):\n assert self.ac.is_unknown\n\n def test_is_failed(self):\n assert not self.ac.is_failed\n\n def test_is_failed_true(self):\n self.ac.exit_status = 255\n assert self.ac.is_failed\n\n def test_is_complete(self):\n assert not self.ac.is_complete\n\n def test_is_complete_true(self):\n self.ac.machine.state = self.ac.COMPLETE\n assert self.ac.is_complete, self.ac.machine.state\n\n def test_is_done(self):\n self.ac.machine.state = self.ac.FAILSTART\n assert self.ac.is_done, self.ac.machine.state\n self.ac.machine.state = self.ac.COMPLETE\n assert self.ac.is_done, self.ac.machine.state\n\n\nclass TestCreateActionCommandFactoryFromConfig(TestCase):\n def test_create_default_action_command_no_config(self):\n config = ()\n factory = actioncommand.create_action_runner_factory_from_config(\n config,\n )\n assert_equal(type(factory), actioncommand.NoActionRunnerFactory)\n\n def test_create_default_action_command(self):\n config = schema.ConfigActionRunner(\n schema.ActionRunnerTypes.none.value,\n None,\n None,\n )\n factory = actioncommand.create_action_runner_factory_from_config(\n config,\n )\n assert type(factory) is actioncommand.NoActionRunnerFactory\n\n def test_create_action_command_with_simple_runner(self):\n status_path, exec_path = \"/tmp/what\", \"/remote/bin\"\n config = schema.ConfigActionRunner(\n schema.ActionRunnerTypes.subprocess.value,\n status_path,\n exec_path,\n )\n factory = actioncommand.create_action_runner_factory_from_config(\n config,\n )\n assert_equal(factory.status_path, status_path)\n assert_equal(factory.exec_path, exec_path)\n\n\nclass TestSubprocessActionRunnerFactory(TestCase):\n @setup\n def setup_factory(self):\n self.status_path = \"status_path\"\n self.exec_path = \"exec_path\"\n self.factory = actioncommand.SubprocessActionRunnerFactory(\n self.status_path,\n self.exec_path,\n )\n\n def test_from_config(self):\n config = mock.Mock()\n runner_factory = actioncommand.SubprocessActionRunnerFactory.from_config(\n config,\n )\n assert_equal(runner_factory.status_path, config.remote_status_path)\n assert_equal(runner_factory.exec_path, config.remote_exec_path)\n\n def test_create(self):\n serializer = mock.create_autospec(actioncommand.StringBufferStore)\n id, command = \"id\", \"do a thing\"\n autospec_method(self.factory.build_command)\n action_command = self.factory.create(id, command, serializer)\n assert_equal(action_command.id, id)\n assert_equal(\n action_command.command,\n self.factory.build_command.return_value,\n )\n assert_equal(action_command.stdout, serializer.open.return_value)\n assert_equal(action_command.stderr, serializer.open.return_value)\n\n def test_build_command_complex_quoting(self):\n id = \"id\"\n command = '/bin/foo -c \"foo\" --foo \"bar\"'\n exec_name = \"action_runner.py\"\n actual = self.factory.build_command(id, command, exec_name)\n assert_equal(\n shlex.split(actual),\n [\n f\"{self.exec_path}/{exec_name}\",\n f\"{self.status_path}/{id}\",\n command,\n id,\n ],\n )\n\n def test_build_stop_action_command(self):\n id, command = \"id\", \"do a thing\"\n autospec_method(self.factory.build_command)\n action_command = self.factory.build_stop_action_command(id, command)\n assert_equal(\n action_command.id,\n f\"{id}.{self.factory.build_command.return_value}\",\n )\n assert_equal(\n action_command.command,\n self.factory.build_command.return_value,\n )\n\n def test__eq__true(self):\n first = actioncommand.SubprocessActionRunnerFactory(\"a\", \"b\")\n second = actioncommand.SubprocessActionRunnerFactory(\"a\", \"b\")\n assert_equal(first, second)\n\n def test__eq__false(self):\n first = actioncommand.SubprocessActionRunnerFactory(\"a\", \"b\")\n second = actioncommand.SubprocessActionRunnerFactory(\"a\", \"c\")\n assert_not_equal(first, second)\n assert_not_equal(first, None)\n assert_not_equal(first, actioncommand.NoActionRunnerFactory)\n" }, { "path": "tests/api/__init__.py", "content": "" }, { "path": "tests/api/adapter_test.py", "content": "import shutil\nimport tempfile\nfrom unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import teardown\nfrom testifycompat import TestCase\nfrom tests import mocks\nfrom tests.assertions import assert_length\nfrom tron import node\nfrom tron import scheduler\nfrom tron.api import adapter\nfrom tron.api.adapter import ActionRunAdapter\nfrom tron.api.adapter import JobRunAdapter\nfrom tron.api.adapter import ReprAdapter\nfrom tron.api.adapter import RunAdapter\nfrom tron.core import actiongraph\nfrom tron.core import actionrun\nfrom tron.core import job\n\n\nclass MockAdapter(ReprAdapter):\n\n field_names = [\"one\", \"two\"]\n translated_field_names = [\"three\", \"four\"]\n\n def get_three(self):\n return 3\n\n def get_four(self):\n return 4\n\n\nclass TestReprAdapter(TestCase):\n @setup\n def setup_adapter(self):\n self.original = mock.Mock(one=1, two=2)\n self.adapter = MockAdapter(self.original)\n\n def test__init__(self):\n assert_equal(self.adapter._obj, self.original)\n assert_equal(self.adapter.fields, MockAdapter.field_names)\n\n def test_get_translation_mapping(self):\n expected = {\n \"three\": self.adapter.get_three,\n \"four\": self.adapter.get_four,\n }\n assert_equal(self.adapter.translators, expected)\n\n def test_get_repr(self):\n expected = dict(one=1, two=2, three=3, four=4)\n assert_equal(self.adapter.get_repr(), expected)\n\n\nclass SampleClassStub:\n def __init__(self):\n self.true_flag = True\n self.false_flag = False\n\n @adapter.toggle_flag(\"true_flag\")\n def expects_true(self):\n return \"This is true\"\n\n @adapter.toggle_flag(\"false_flag\")\n def expects_false(self):\n return \"This is false\"\n\n\nclass TestToggleFlag(TestCase):\n @setup\n def setup_stub(self):\n self.stub = SampleClassStub()\n\n def test_toggle_flag_true(self):\n assert_equal(self.stub.expects_true(), \"This is true\")\n\n def test_toggle_flag_false(self):\n assert not self.stub.expects_false()\n\n\nclass TestRunAdapter(TestCase):\n @setup\n def setup_adapter(self):\n self.original = mock.Mock()\n self.adapter = RunAdapter(self.original)\n\n def test_get_state(self):\n assert_equal(self.adapter.get_state(), self.original.state)\n\n @mock.patch(\"tron.api.adapter.NodeAdapter\", autospec=True)\n def test_get_node(self, mock_node_adapter):\n assert_equal(\n self.adapter.get_node(),\n mock_node_adapter.return_value.get_repr.return_value,\n )\n mock_node_adapter.assert_called_with(self.original.node)\n\n def test_get_duration(self):\n self.original.start_time = None\n assert_equal(self.adapter.get_duration(), \"\")\n\n\nclass TestActionRunAdapter(TestCase):\n @setup\n def setup_adapter(self):\n self.temp_dir = tempfile.mkdtemp()\n self.action_run = mock.MagicMock()\n self.job_run = mock.MagicMock()\n self.adapter = ActionRunAdapter(self.action_run, self.job_run, 4)\n\n @teardown\n def teardown_adapter(self):\n shutil.rmtree(self.temp_dir)\n\n def test__init__(self):\n assert_equal(self.adapter.max_lines, 4)\n assert_equal(self.adapter.job_run, self.job_run)\n assert_equal(self.adapter._obj, self.action_run)\n\n def test_get_repr(self):\n result = self.adapter.get_repr()\n assert_equal(result[\"command\"], self.action_run.rendered_command)\n\n\nclass TestActionRunGraphAdapter(TestCase):\n @setup\n def setup_adapter(self):\n self.ar1 = mock.MagicMock(action_name=\"a1\")\n self.ar2 = mock.MagicMock(action_name=\"a2\")\n self.a1 = mock.MagicMock()\n self.a2 = mock.MagicMock()\n self.a1.name = \"a1\"\n self.a2.name = \"a2\"\n self.action_runs = mock.create_autospec(\n actionrun.ActionRunCollection,\n action_graph=actiongraph.ActionGraph(\n {\n \"a1\": self.a1,\n \"a2\": self.a2,\n },\n {\"a1\": set(), \"a2\": {\"a1\"}},\n {\"a1\": set(), \"a2\": set()},\n ),\n )\n self.adapter = adapter.ActionRunGraphAdapter(self.action_runs)\n self.action_runs.__iter__.return_value = [self.ar1, self.ar2]\n\n def test_get_repr(self):\n result = self.adapter.get_repr()\n assert len(result) == 2\n assert self.ar1.id == result[0][\"id\"]\n assert [\"a1\"] == result[1][\"dependencies\"]\n assert self.ar1.rendered_command == result[0][\"command\"]\n assert self.ar1.command_config.command == result[0][\"raw_command\"]\n\n\nclass TestJobRunAdapter(TestCase):\n @setup\n def setup_adapter(self):\n action_runs = mock.MagicMock()\n action_runs.__iter__.return_value = iter([mock.Mock(), mock.Mock()])\n self.job_run = mock.Mock(\n action_runs=action_runs,\n action_graph=mocks.MockActionGraph(),\n )\n self.adapter = JobRunAdapter(self.job_run, include_action_runs=True)\n\n def test__init__(self):\n assert self.adapter.include_action_runs\n\n def test_get_runs(self):\n with mock.patch(\"tron.api.adapter.ActionRunAdapter\", autospec=True):\n assert_length(self.adapter.get_runs(), 2)\n\n def test_get_runs_without_action_runs(self):\n self.adapter.include_action_runs = False\n assert_equal(self.adapter.get_runs(), None)\n\n\nclass TestNodeAdapter(TestCase):\n @setup\n def setup_adapter(self):\n self.node = mock.create_autospec(node.Node)\n self.adapter = adapter.NodeAdapter(self.node)\n\n def test_repr(self):\n result = self.adapter.get_repr()\n assert_equal(result[\"hostname\"], self.node.hostname)\n assert_equal(result[\"username\"], self.node.username)\n\n\nclass TestNodePoolAdapter(TestCase):\n @setup\n def setup_adapter(self):\n self.pool = mock.create_autospec(node.NodePool)\n self.adapter = adapter.NodePoolAdapter(self.pool)\n\n @mock.patch(\"tron.api.adapter.adapt_many\", autospec=True)\n def test_repr(self, mock_many):\n result = self.adapter.get_repr()\n assert_equal(result[\"name\"], self.pool.get_name.return_value)\n mock_many.assert_called_with(\n adapter.NodeAdapter,\n self.pool.get_nodes.return_value,\n )\n\n\nclass TestJobIndexAdapter(TestCase):\n @setup\n def setup_adapter(self):\n self.job = mock.create_autospec(job.Job)\n self.adapter = adapter.JobIndexAdapter(self.job)\n\n def test_repr(self):\n result = self.adapter.get_repr()\n self.job.get_runs.assert_called_with()\n runs = self.job.get_runs.return_value\n runs.get_newest.assert_called_with()\n expected = {\n \"name\": self.job.get_name.return_value,\n \"actions\": [],\n }\n assert_equal(result, expected)\n\n def test_get_actions(self):\n action_run = mock.Mock()\n job_run = self.job.get_runs.return_value.get_newest.return_value\n job_run.action_runs.__iter__.return_value = [action_run]\n result = self.adapter.get_actions()\n expected = {\n \"name\": action_run.action_name,\n \"command\": action_run.command_config.command,\n }\n assert_equal(result, [expected])\n\n def test_get_actions_no_runs(self):\n self.job.get_runs.return_value.get_newest.return_value = None\n result = self.adapter.get_actions()\n assert_equal(result, [])\n\n\nclass TestSchedulerAdapter(TestCase):\n @setup\n def setup_adapter(self):\n self.scheduler = mock.create_autospec(scheduler.GeneralScheduler)\n self.adapter = adapter.SchedulerAdapter(self.scheduler)\n\n @mock.patch(\"tron.api.adapter.scheduler.get_jitter_str\", autospec=True)\n def test_repr(self, mock_get_jitter):\n result = self.adapter.get_repr()\n expected = {\n \"type\": self.scheduler.get_name.return_value,\n \"value\": self.scheduler.get_value.return_value,\n \"jitter\": mock_get_jitter.return_value,\n }\n assert_equal(result, expected)\n mock_get_jitter.assert_called_with(self.scheduler.get_jitter())\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/api/auth_test.py", "content": "from unittest.mock import MagicMock\nfrom unittest.mock import patch\n\nimport pytest\nfrom twisted.web.server import Request\n\nfrom tron.api.auth import AuthorizationFilter\nfrom tron.api.auth import AuthorizationOutcome\n\n\n@pytest.fixture\ndef mock_auth_filter():\n with patch(\"tron.api.auth.requests\"):\n yield AuthorizationFilter(\"http://localhost:31337/whatever\", True)\n\n\ndef mock_request(path: str, token: str, method: str):\n res = MagicMock(spec=Request, path=path.encode(), method=method.encode())\n res.getHeader.return_value = token\n return res\n\n\ndef test_is_request_authorized(mock_auth_filter):\n mock_auth_filter.session.post.return_value.json.return_value = {\n \"result\": {\"allowed\": True, \"reason\": \"User allowed\"}\n }\n assert mock_auth_filter.is_request_authorized(\n mock_request(\"/api/jobs/foobar.run.2\", \"aaa.bbb.ccc\", \"get\")\n ) == AuthorizationOutcome(True, \"User allowed\")\n mock_auth_filter.session.post.assert_called_once_with(\n url=\"http://localhost:31337/whatever\",\n json={\n \"input\": {\n \"path\": \"/api/jobs/foobar.run.2\",\n \"backend\": \"tron\",\n \"token\": \"aaa.bbb.ccc\",\n \"method\": \"get\",\n \"service\": \"foobar\",\n }\n },\n timeout=2,\n )\n\n\ndef test_is_request_authorized_fail(mock_auth_filter):\n mock_auth_filter.session.post.side_effect = Exception\n assert mock_auth_filter.is_request_authorized(\n mock_request(\"/allowed\", \"eee.ddd.fff\", \"get\")\n ) == AuthorizationOutcome(False, \"Auth backend error\")\n\n\ndef test_is_request_authorized_malformed(mock_auth_filter):\n mock_auth_filter.session.post.return_value.json.return_value = {\"foo\": \"bar\"}\n assert mock_auth_filter.is_request_authorized(\n mock_request(\"/allowed\", \"eee.ddd.fff\", \"post\")\n ) == AuthorizationOutcome(False, \"Malformed auth response\")\n\n\ndef test_is_request_authorized_no_enforce(mock_auth_filter):\n mock_auth_filter.session.post.return_value.json.return_value = {\n \"result\": {\"allowed\": False, \"reason\": \"Missing token\"}\n }\n with patch.object(mock_auth_filter, \"enforce\", False):\n assert mock_auth_filter.is_request_authorized(mock_request(\"/foobar\", \"\", \"post\")) == AuthorizationOutcome(\n True, \"Auth dry-run\"\n )\n\n\ndef test_is_request_authorized_disabled(mock_auth_filter):\n mock_auth_filter.session.post.return_value.json.return_value = {\n \"result\": {\"allowed\": False, \"reason\": \"Missing token\"}\n }\n with patch.object(mock_auth_filter, \"endpoint\", None):\n assert mock_auth_filter.is_request_authorized(mock_request(\"/buzz\", \"\", \"post\")) == AuthorizationOutcome(\n True, \"Auth not enabled\"\n )\n\n\n@pytest.mark.parametrize(\n \"path,expected\",\n (\n (\"/api/jobs/someservice.instance/110/run\", \"someservice\"),\n (\"/api/jobs/someweirdservice/110/run\", \"someweirdservice\"),\n (\"/api/jobs/\", None),\n (\"/api\", None),\n ),\n)\ndef test_extract_service_from_path(path, expected):\n assert AuthorizationFilter._extract_service_from_path(path) == expected\n" }, { "path": "tests/api/controller_test.py", "content": "from unittest import mock\n\nimport pytest\n\nfrom tron import mcp\nfrom tron.api import controller\nfrom tron.api.controller import ConfigController\nfrom tron.api.controller import EventsController\nfrom tron.api.controller import InvalidCommandForActionState\nfrom tron.api.controller import JobCollectionController\nfrom tron.api.controller import UnknownCommandError\nfrom tron.config import ConfigError\nfrom tron.config import manager\nfrom tron.core import actionrun\nfrom tron.core import jobrun\nfrom tron.core.job_collection import JobCollection\nfrom tron.core.job_scheduler import JobScheduler\n\n\nclass TestJobCollectionController:\n @pytest.fixture(autouse=True)\n def setup_controller(self):\n self.collection = mock.create_autospec(\n JobCollection,\n enable=mock.Mock(),\n disable=mock.Mock(),\n )\n self.controller = JobCollectionController(self.collection)\n\n def test_handle_command_unknown(self):\n with pytest.raises(UnknownCommandError):\n self.controller.handle_command(\"enableall\")\n self.controller.handle_command(\"disableall\")\n\n def test_handle_command_move_non_existing_job(self):\n self.collection.get_names.return_value = []\n result = self.controller.handle_command(\n \"move\",\n old_name=\"old.test\",\n new_name=\"new.test\",\n )\n assert \"doesn't exist\" in result\n\n def test_handle_command_move_to_existing_job(self):\n self.collection.get_names.return_value = [\"old.test\", \"new.test\"]\n result = self.controller.handle_command(\n \"move\",\n old_name=\"old.test\",\n new_name=\"new.test\",\n )\n assert \"exists already\" in result\n\n def test_handle_command_move(self):\n self.collection.get_names.return_value = [\"old.test\"]\n result = self.controller.handle_command(\n \"move\",\n old_name=\"old.test\",\n new_name=\"new.test\",\n )\n assert \"Error\" not in result\n\n\nclass TestActionRunController:\n @pytest.fixture(autouse=True)\n def setup_controller(self):\n self.action_run = mock.create_autospec(\n actionrun.ActionRun,\n cancel=mock.Mock(),\n )\n self.job_run = mock.create_autospec(jobrun.JobRun)\n self.job_run.is_scheduled = False\n self.controller = controller.ActionRunController(\n self.action_run,\n self.job_run,\n )\n self.job_run.action_runs.cleanup_action_run = None\n\n def test_handle_command_start_failed(self):\n self.job_run.is_scheduled = True\n result = self.controller.handle_command(\"start\")\n assert not self.action_run.start.mock_calls\n assert \"cannot be started\" in result\n\n def test_handle_command_recover_failed(self):\n self.action_run.is_unknown = False\n result = self.controller.handle_command(\"recover\")\n assert not self.action_run.recover.mock_calls\n assert \"cannot be recovered\" in result\n\n def test_handle_command_mapped_command(self):\n result = self.controller.handle_command(\"cancel\")\n self.action_run.cancel.assert_called_with()\n assert \"now in state\" in result\n\n def test_handle_command_mapped_command_failed(self):\n self.action_run.cancel.return_value = False\n with pytest.raises(InvalidCommandForActionState):\n self.controller.handle_command(\"cancel\")\n\n def test_handle_termination_not_implemented(self):\n self.action_run.stop.side_effect = NotImplementedError\n result = self.controller.handle_termination(\"stop\")\n assert \"Failed to stop\" in result\n\n def test_handle_termination_success_without_extra_msg(self):\n self.action_run.kill.return_value = None\n result = self.controller.handle_termination(\"kill\")\n assert \"Attempting to kill\" in result\n\n def test_handle_termination_success_with_extra_msg(self):\n self.action_run.kill.return_value = \"Warning Message\"\n result = self.controller.handle_termination(\"kill\")\n assert \"Attempting to kill\" in result\n assert \"Warning Message\" in result\n\n def test_handle_retry_default(self):\n self.controller.handle_command(\"retry\")\n self.action_run.retry.assert_called_once_with(original_command=True)\n\n def test_handle_retry_new_command(self):\n self.controller.handle_command(\"retry\", use_latest_command=True)\n self.action_run.retry.assert_called_once_with(original_command=False)\n\n\nclass TestJobRunController:\n @pytest.fixture(autouse=True)\n def setup_controller(self):\n self.job_run = mock.create_autospec(\n jobrun.JobRun,\n run_time=mock.Mock(),\n cancel=mock.Mock(),\n )\n self.job_scheduler = mock.create_autospec(JobScheduler)\n self.controller = controller.JobRunController(\n self.job_run,\n self.job_scheduler,\n )\n\n def test_handle_command_restart(self):\n self.controller.handle_command(\"restart\")\n self.job_scheduler.manual_start.assert_called_with(\n self.job_run.run_time,\n )\n\n def test_handle_mapped_command(self):\n result = self.controller.handle_command(\"start\")\n self.job_run.start.assert_called_with()\n assert \"now in state\" in result\n\n def test_handle_mapped_command_failure(self):\n self.job_run.cancel.return_value = False\n result = self.controller.handle_command(\"cancel\")\n self.job_run.cancel.assert_called_with()\n assert \"Failed to cancel\" in result\n\n\nclass TestJobController:\n @pytest.fixture(autouse=True)\n def setup_controller(self):\n self.job_scheduler = mock.create_autospec(JobScheduler)\n self.controller = controller.JobController(self.job_scheduler)\n\n def test_handle_command_enable(self):\n self.controller.handle_command(\"enable\")\n self.job_scheduler.enable.assert_called_with()\n\n def test_handle_command_disable(self):\n self.controller.handle_command(\"disable\")\n self.job_scheduler.disable.assert_called_with()\n\n def test_handle_command_start(self):\n run_time = mock.Mock()\n self.controller.handle_command(\"start\", run_time)\n self.job_scheduler.manual_start.assert_called_with(run_time=run_time)\n\n\nclass TestConfigController:\n @pytest.fixture(autouse=True)\n def setup_controller(self):\n self.mcp = mock.create_autospec(mcp.MasterControlProgram)\n self.manager = mock.create_autospec(manager.ConfigManager)\n self.mcp.get_config_manager.return_value = self.manager\n self.controller = ConfigController(self.mcp)\n\n def test_get_config_content_new(self):\n self.manager.__contains__.return_value = False\n content = self.controller._get_config_content(\"name\")\n assert content == self.controller.DEFAULT_NAMED_CONFIG\n assert not self.manager.read_raw_config.call_count\n\n def test_get_config_content_old(self):\n self.manager.__contains__.return_value = True\n name = \"the_name\"\n content = self.controller._get_config_content(name)\n assert content == self.manager.read_raw_config.return_value\n self.manager.read_raw_config.assert_called_with(name)\n\n def test_read_config(self):\n self.manager.__contains__.return_value = True\n name = \"MASTER\"\n resp = self.controller.read_config(name)\n self.manager.read_raw_config.assert_called_with(name)\n self.manager.get_hash.assert_called_with(name)\n assert resp[\"config\"] == self.manager.read_raw_config.return_value\n assert resp[\"hash\"] == self.manager.get_hash.return_value\n\n def test_update_config(self):\n name, content, config_hash = \"foo_namespace\", mock.Mock(), mock.Mock()\n self.manager.get_hash.return_value = config_hash\n assert not self.controller.update_config(name, content, config_hash)\n self.manager.get_hash.assert_called_with(name)\n self.manager.write_config.assert_called_with(name, content)\n self.mcp.reconfigure.assert_called_with(name)\n\n def test_update_config_failure(self):\n name, content, old_content, config_hash = (\n \"foo_namespace\",\n mock.Mock(),\n mock.Mock(),\n mock.Mock(),\n )\n self.manager.get_hash.return_value = config_hash\n self.manager.write_config.side_effect = [ConfigError(\"It broke\"), None]\n self.controller.read_config = mock.Mock(return_value={\"config\": old_content})\n error = self.controller.update_config(\n name,\n content,\n config_hash,\n )\n assert error == \"It broke\"\n self.manager.write_config.call_args_list = [\n (name, content),\n (name, old_content),\n ]\n assert self.mcp.reconfigure.call_count == 1\n self.mcp.reconfigure.assert_called_with(name)\n\n def test_update_config_hash_mismatch(self):\n name, content, config_hash = \"foo_namespace\", mock.Mock(), mock.Mock()\n error = self.controller.update_config(name, content, config_hash)\n assert error == \"Configuration has changed. Please try again.\"\n\n def test_delete_config(self):\n name, content, config_hash = \"foo_namespace\", \"\", mock.Mock()\n self.manager.get_hash.return_value = config_hash\n assert not self.controller.delete_config(name, content, config_hash)\n self.manager.delete_config.assert_called_with(name)\n self.mcp.reconfigure.assert_called_with(name)\n self.manager.get_hash.assert_called_with(name)\n\n def test_delete_config_failure(self):\n name, content, config_hash = \"foo_namespace\", \"\", mock.Mock()\n self.manager.get_hash.return_value = config_hash\n self.manager.delete_config.side_effect = Exception(\"some error\")\n error = self.controller.delete_config(name, content, config_hash)\n assert error\n self.manager.delete_config.assert_called_with(name)\n assert not self.mcp.reconfigure.call_count\n\n def test_delete_config_hash_mismatch(self):\n name, content, config_hash = \"foo_namespace\", \"\", mock.Mock()\n error = self.controller.delete_config(name, content, config_hash)\n assert error == \"Configuration has changed. Please try again.\"\n\n def test_delete_config_content_not_empty(self):\n name, content, config_hash = \"foo_namespace\", \"\", mock.Mock()\n error = self.controller.delete_config(name, content, config_hash)\n assert error\n\n def test_get_namespaces(self):\n result = self.controller.get_namespaces()\n self.manager.get_namespaces.assert_called_with()\n assert result == self.manager.get_namespaces.return_value\n\n\nclass TestEventsController:\n @pytest.fixture(autouse=True)\n def setup(self):\n with mock.patch(\"tron.api.controller.EventBus\", autospec=True) as eb:\n eb.instance = mock.Mock()\n self.eventbus = eb\n self.controller = EventsController()\n yield\n\n def test_info(self):\n self.eventbus.instance = None\n assert self.controller.info() == dict(error=\"EventBus disabled\")\n\n self.eventbus.instance = mock.Mock()\n assert self.controller.info() == dict(response=self.eventbus.instance.event_log)\n\n def test_publish(self):\n event = mock.Mock()\n self.eventbus.instance = None\n self.eventbus.has_event.return_value = True\n self.eventbus.publish.return_value = False\n\n assert self.controller.info() == dict(error=\"EventBus disabled\")\n assert len(self.eventbus.publish.mock_calls) == 0\n\n self.eventbus.instance = mock.Mock()\n assert self.controller.publish(event) == dict(response=f\"event {event} already published\")\n assert len(self.eventbus.publish.mock_calls) == 0\n\n self.eventbus.has_event.return_value = False\n assert self.controller.publish(event) == dict(error=f\"could not publish {event}\")\n assert len(self.eventbus.publish.mock_calls) == 1\n\n self.eventbus.publish.return_value = True\n assert self.controller.publish(event) == dict(response=\"OK\")\n assert len(self.eventbus.publish.mock_calls) == 2\n\n def test_discard(self):\n event = mock.Mock()\n self.eventbus.instance = None\n self.eventbus.discard.return_value = False\n\n assert self.controller.info() == dict(error=\"EventBus disabled\")\n assert len(self.eventbus.discard.mock_calls) == 0\n\n self.eventbus.instance = mock.Mock()\n assert self.controller.discard(event) == dict(error=f\"could not discard {event}\")\n assert len(self.eventbus.discard.mock_calls) == 1\n\n self.eventbus.discard.return_value = True\n assert self.controller.discard(event) == dict(response=\"OK\")\n assert len(self.eventbus.discard.mock_calls) == 2\n" }, { "path": "tests/api/requestargs_test.py", "content": "import datetime\nfrom unittest.mock import MagicMock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tron.api.requestargs import get_bool\nfrom tron.api.requestargs import get_datetime\nfrom tron.api.requestargs import get_integer\nfrom tron.api.requestargs import get_string\n\n\nclass TestRequestArgs(TestCase):\n @setup\n def setup_args(self):\n self.args = {\n b\"number\": [b\"123\"],\n b\"string\": [b\"astring\"],\n b\"boolean\": [b\"1\"],\n b\"datetime\": [b\"2012-03-14 15:09:26\"],\n }\n self.datetime = datetime.datetime(2012, 3, 14, 15, 9, 26)\n self.request = MagicMock(args=self.args)\n\n def _add_arg(self, name, value):\n name = name.encode()\n value = value.encode()\n if name not in self.args:\n self.args[name] = []\n self.args[name].append(value)\n\n def test_get_integer_valid_int(self):\n self._add_arg(\"number\", \"5\")\n assert_equal(get_integer(self.request, \"number\"), 123)\n\n def test_get_integer_invalid_int(self):\n self._add_arg(\"nan\", \"beez\")\n assert not get_integer(self.request, \"nan\")\n\n def test_get_integer_missing(self):\n assert not get_integer(self.request, \"missing\")\n\n def test_get_string(self):\n self._add_arg(\"string\", \"bogus\")\n assert_equal(get_string(self.request, \"string\"), \"astring\")\n\n def test_get_string_missing(self):\n assert not get_string(self.request, \"missing\")\n\n def test_get_bool(self):\n assert get_bool(self.request, \"boolean\")\n\n def test_get_bool_false(self):\n self._add_arg(\"false\", \"0\")\n assert not get_bool(self.request, \"false\")\n\n def test_get_bool_missing(self):\n assert not get_bool(self.request, \"missing\")\n\n def test_get_datetime_valid(self):\n assert_equal(get_datetime(self.request, \"datetime\"), self.datetime)\n\n def test_get_datetime_invalid(self):\n self._add_arg(\"nope\", \"2012-333-4\")\n assert not get_datetime(self.request, \"nope\")\n\n def test_get_datetime_missing(self):\n assert not get_datetime(self.request, \"missing\")\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/api/resource_test.py", "content": "\"\"\"\nTest cases for the web services interface to tron\n\"\"\"\nfrom unittest import mock\nfrom unittest.mock import MagicMock\n\nimport pytest\nimport staticconf.testing\nimport twisted.web.http\nimport twisted.web.resource\nimport twisted.web.server\nfrom twisted.web import http\n\nfrom tests.assertions import assert_call\nfrom tests.testingutils import autospec_method\nfrom tron import __version__\nfrom tron import mcp\nfrom tron import node\nfrom tron.api import controller\nfrom tron.core import job\nfrom tron.core import jobrun\nfrom tron.core.job_collection import JobCollection\nfrom tron.core.job_scheduler import JobScheduler\n\nwith mock.patch(\n \"tron.api.async_resource.AsyncResource.bounded\",\n lambda fn: fn,\n autospec=None,\n):\n with mock.patch(\n \"tron.api.async_resource.AsyncResource.exclusive\",\n lambda fn: fn,\n autospec=None,\n ):\n from tron.api import resource as www\n\nREQUEST = twisted.web.server.Request(mock.Mock(), None)\nREQUEST.childLink = lambda val: \"/jobs/%s\" % val\n\n\ndef build_request(**kwargs):\n args = {k.encode(): [v.encode()] for k, v in kwargs.items()}\n return mock.create_autospec(twisted.web.server.Request, args=args)\n\n\n@pytest.fixture\ndef mock_request():\n return build_request()\n\n\n@pytest.fixture\ndef mock_respond():\n with mock.patch(\n \"tron.api.resource.respond\",\n autospec=True,\n ) as mock_respond:\n mock_respond.side_effect = lambda request, response, code=None: response\n yield mock_respond\n\n\n@pytest.mark.usefixtures(\"mock_respond\")\nclass WWWTestCase:\n \"\"\"Patch www.response to not json encode.\"\"\"\n\n pass\n\n\n@pytest.mark.parametrize(\n \"response,code,expected_code\",\n [\n (\"a_string\", None, 200),\n ([{\"a\": \"list\"}], None, 200),\n ({\"a\": \"dict\"}, None, 200),\n ({\"a\": \"dict\"}, 501, 501),\n ({\"error\": \"something went wrong\"}, None, 500),\n ],\n)\ndef test_respond(response, code, expected_code):\n request = build_request()\n www.respond(request, response, code=code)\n request.setResponseCode.assert_called_once_with(expected_code)\n\n\nclass TestHandleCommand:\n @pytest.fixture\n def mock_respond(self, mock_respond):\n # in this test case, we don't want a side effect\n mock_respond.side_effect = None\n return mock_respond\n\n def test_handle_command_unknown(self, mock_respond):\n command = \"the command\"\n request = build_request(command=command)\n mock_controller, obj = mock.Mock(), mock.Mock()\n error = controller.UnknownCommandError()\n mock_controller.handle_command.side_effect = error\n www.handle_command(request, mock_controller, obj)\n mock_controller.handle_command.assert_called_with(command)\n mock_respond.assert_called_with(\n request=request,\n response={\"error\": f\"Unknown command '{command}' for '{obj}'\"},\n code=http.NOT_IMPLEMENTED,\n )\n\n def test_handle_command(self, mock_respond):\n command = \"the command\"\n request = build_request(command=command)\n mock_controller, obj = mock.Mock(), mock.Mock()\n www.handle_command(request, mock_controller, obj)\n mock_controller.handle_command.assert_called_with(command)\n mock_respond.assert_called_with(\n request=request,\n response={\"result\": mock_controller.handle_command.return_value},\n )\n\n def test_handle_command_error(self, mock_respond):\n command = \"the command\"\n request = build_request(command=command)\n mock_controller, obj = mock.Mock(), mock.Mock()\n error = Exception(\"uncaught exception\")\n mock_controller.handle_command.side_effect = error\n www.handle_command(request, mock_controller, obj)\n mock_controller.handle_command.assert_called_with(command)\n mock_respond.assert_called_with(request=request, response={\"error\": mock.ANY})\n\n\nclass TestActionRunResource(WWWTestCase):\n @pytest.fixture(autouse=True)\n def setup_resource(self):\n self.job_run = mock.MagicMock()\n self.action_run = mock.MagicMock(output_path=[\"one\"])\n with mock.patch(\"tron.config.static_config.load_yaml_file\", autospec=True,), mock.patch(\n \"tron.config.static_config.build_configuration_watcher\",\n autospec=True,\n ):\n self.resource = www.ActionRunResource(self.action_run, self.job_run)\n\n def test_render_GET(self, mock_respond):\n request = build_request()\n mock_config = {\"logging.max_lines_to_display\": 1000}\n mock_configuration = staticconf.testing.MockConfiguration(mock_config, namespace=\"tron\")\n with mock_configuration:\n response = self.resource.render_GET(request)\n assert response[\"id\"] == self.resource.action_run.id\n\n\nclass TestJobrunResource(WWWTestCase):\n @pytest.fixture(autouse=True)\n def setup_resource(self):\n self.job_run = mock.MagicMock()\n self.job_scheduler = mock.Mock()\n self.resource = www.JobRunResource(self.job_run, self.job_scheduler)\n\n def test_render_GET(self, mock_request):\n response = self.resource.render_GET(mock_request)\n assert response[\"id\"] == self.job_run.id\n\n\nclass TestApiRootResource(WWWTestCase):\n @pytest.fixture(autouse=True)\n def setup_resource(self):\n self.mcp = mock.create_autospec(mcp.MasterControlProgram)\n self.resource = www.ApiRootResource(self.mcp)\n\n def test__init__(self):\n expected_children = [\n b\"jobs\",\n b\"config\",\n b\"metrics\",\n b\"status\",\n b\"events\",\n b\"prom-metrics\",\n b\"\",\n ]\n assert set(expected_children) == set(self.resource.children)\n\n def test_render_GET(self):\n expected_keys = [\n \"jobs\",\n \"namespaces\",\n ]\n response = self.resource.render_GET(build_request())\n assert set(response.keys()) == set(expected_keys)\n self.mcp.get_job_collection().get_jobs.assert_called_with()\n\n\nclass TestRootResource(WWWTestCase):\n @pytest.fixture(autouse=True)\n def setup_resource(self):\n self.web_path = \"/bogus/path\"\n self.mcp = mock.create_autospec(mcp.MasterControlProgram)\n self.resource = www.RootResource(self.mcp, self.web_path)\n\n def test_render_GET(self):\n request = build_request()\n response = self.resource.render_GET(request)\n assert response == 1\n assert request.redirect.call_count == 1\n request.finish.assert_called_with()\n\n def test_get_children(self):\n assert set(self.resource.children) == {b\"api\", b\"web\", b\"\"}\n\n\nclass TestActionRunHistoryResource(WWWTestCase):\n @pytest.fixture(autouse=True)\n def setup_resource(self):\n self.action_runs = [mock.MagicMock(), mock.MagicMock()]\n self.resource = www.ActionRunHistoryResource(self.action_runs)\n\n def test_render_GET(self, request):\n response = self.resource.render_GET(request)\n assert len(response) == len(self.action_runs)\n\n\nclass TestJobCollectionResource(WWWTestCase):\n @pytest.fixture(autouse=True)\n def setup_resource(self):\n job_collection = mock.create_autospec(JobCollection)\n job_collection.get_by_name = lambda name: name if name == \"testname\" else None\n self.resource = www.JobCollectionResource(job_collection)\n\n def test_render_GET(self):\n self.resource.get_data = MagicMock()\n result = self.resource.render_GET(REQUEST)\n assert_call(self.resource.get_data, 0, False, False, True, True)\n assert \"jobs\" in result\n\n def test_getChild(self):\n child = self.resource.getChild(b\"testname\", mock.Mock())\n assert isinstance(child, www.JobResource)\n\n def test_getChild_missing_job(self):\n child = self.resource.getChild(b\"bar\", mock.Mock())\n assert isinstance(child, www.ErrorResource)\n\n\nclass TestJobResource(WWWTestCase):\n @pytest.fixture(autouse=True)\n def setup_resource(self):\n self.job_scheduler = mock.create_autospec(JobScheduler)\n self.job_runs = mock.create_autospec(jobrun.JobRunCollection)\n self.job = mock.create_autospec(\n job.Job,\n runs=self.job_runs,\n all_nodes=False,\n allow_overlap=True,\n queueing=True,\n action_graph=mock.MagicMock(),\n scheduler=mock.Mock(),\n node_pool=mock.create_autospec(node.NodePool),\n max_runtime=mock.Mock(),\n expected_runtime=mock.MagicMock(),\n )\n self.job.get_name.return_value = \"foo\"\n self.job_scheduler.get_job.return_value = self.job\n self.job_scheduler.get_job_runs.return_value = self.job_runs\n self.resource = www.JobResource(self.job_scheduler)\n\n def test_render_GET(self, mock_request):\n result = self.resource.render_GET(mock_request)\n assert result[\"name\"] == self.job_scheduler.get_job().get_name()\n\n def test_get_run_from_identifier_HEAD(self):\n job_run = self.resource.get_run_from_identifier(\"HEAD\")\n self.job_scheduler.get_job_runs.assert_called_with()\n assert job_run == self.job_runs.get_newest.return_value\n\n def test_get_run_from_identifier_number(self):\n job_run = self.resource.get_run_from_identifier(\"3\")\n self.job_scheduler.get_job_runs.assert_called_with()\n assert job_run == self.job_runs.get_run_by_num.return_value\n self.job_runs.get_run_by_num.assert_called_with(3)\n\n def test_get_run_from_identifier_negative_index(self):\n job_run = self.resource.get_run_from_identifier(\"-2\")\n assert job_run == self.job_runs.get_run_by_index.return_value\n self.job_runs.get_run_by_index.assert_called_with(-2)\n\n def test_getChild(self):\n autospec_method(self.resource.get_run_from_identifier)\n identifier = b\"identifier\"\n resource = self.resource.getChild(identifier, None)\n assert resource.job_run == self.resource.get_run_from_identifier.return_value\n\n def test_getChild_action_run_history(self):\n autospec_method(\n self.resource.get_run_from_identifier,\n return_value=None,\n )\n action_name = \"action_name\"\n action_runs = [mock.Mock(), mock.Mock()]\n self.job.action_graph.names.return_value = [action_name]\n self.job.runs.get_action_runs.return_value = action_runs\n resource = self.resource.getChild(action_name, None)\n assert resource.__class__ == www.ActionRunHistoryResource\n assert resource.action_runs == action_runs\n\n\nclass TestConfigResource:\n @pytest.fixture(autouse=True)\n def setup_resource(self):\n self.mcp = mock.create_autospec(mcp.MasterControlProgram)\n self.resource = www.ConfigResource(self.mcp)\n self.controller = self.resource.controller = mock.create_autospec(\n controller.ConfigController,\n )\n\n def test_render_GET(self, mock_respond):\n name = \"the_name\"\n request = build_request(name=name)\n self.resource.render_GET(request)\n self.controller.read_config.assert_called_with(name)\n mock_respond.assert_called_with(\n request=request,\n response=self.resource.controller.read_config.return_value,\n )\n\n def test_render_POST_update(self, mock_respond):\n name, config, hash = \"the_name\", \"config\", \"hash\"\n request = build_request(name=name, config=config, hash=hash)\n self.resource.render_POST(request)\n self.resource.controller.update_config.assert_called_with(name, config, hash)\n expected_response = {\n \"status\": \"Active\",\n \"error\": self.resource.controller.update_config.return_value,\n }\n mock_respond.assert_called_with(request=request, response=expected_response)\n\n def test_render_POST_delete(self, mock_respond):\n name, config, hash = \"the_name\", \"\", \"\"\n request = build_request(name=name, config=config, hash=hash)\n self.resource.render_POST(request)\n self.resource.controller.delete_config.assert_called_with(name, config, hash)\n expected_response = {\n \"status\": \"Active\",\n \"error\": self.resource.controller.delete_config.return_value,\n }\n mock_respond.assert_called_with(request=request, response=expected_response)\n\n\nclass TestStatusResource:\n def test_render_GET(self, request, mock_respond):\n self.mcp = mock.create_autospec(mcp.MasterControlProgram)\n self.mcp.boot_time = 999\n resource = www.StatusResource(self.mcp)\n resource.render_GET(request)\n expected_response = {\n \"status\": \"I'm alive.\",\n \"version\": __version__,\n \"boot_time\": self.mcp.boot_time,\n }\n mock_respond.assert_called_with(\n request=request,\n response=expected_response,\n )\n\n\nclass TestMetricsResource:\n @mock.patch(\"tron.api.resource.view_all_metrics\", autospec=True)\n def test_render_GET(self, mock_view_metrics, request, mock_respond):\n resource = www.MetricsResource()\n resource.render_GET(request)\n mock_respond.assert_called_with(\n request=request,\n response=mock_view_metrics.return_value,\n )\n\n\nclass TestTronSite:\n @mock.patch(\"tron.api.resource.meter\", autospec=True)\n def test_log_request(self, mock_meter):\n site = www.TronSite.create(\n mock.create_autospec(mcp.MasterControlProgram),\n \"webpath\",\n )\n request = mock.Mock(code=500)\n site.log(request)\n assert mock_meter.call_count == 1\n" }, { "path": "tests/assertions.py", "content": "\"\"\"\n Assertions for testify.\n\"\"\"\nfrom testifycompat import assert_in\nfrom testifycompat import assert_not_reached\n\n\ndef assert_raises(expected_exception_class, callable_obj, *args, **kwargs):\n \"\"\"Returns the exception if the callable raises expected_exception_class\"\"\"\n try:\n callable_obj(*args, **kwargs)\n except expected_exception_class as e:\n # we got the expected exception\n return e\n assert_not_reached(\n \"No exception was raised (expected %s)\" % expected_exception_class,\n )\n\n\ndef assert_length(sequence, expected, msg=None):\n \"\"\"Assert that a sequence or iterable has an expected length.\"\"\"\n msg = msg or \"%(sequence)s has length %(length)s expected %(expected)s\"\n length = len(list(sequence))\n assert length == expected, msg % locals()\n\n\ndef assert_call(mock, call_idx, *args, **kwargs):\n \"\"\"Assert that a function was called on mock with the correct args.\"\"\"\n actual = mock.mock_calls[call_idx] if mock.mock_calls else None\n msg = f\"Call {call_idx} expected {(args, kwargs)}, was {actual}\"\n assert actual == (args, kwargs), msg\n\n\ndef assert_mock_calls(expected, mock_calls):\n \"\"\"Assert that all expected calls are in the list of mock_calls.\"\"\"\n for expected_call in expected:\n assert_in(expected_call, mock_calls)\n" }, { "path": "tests/bin/__init__.py", "content": "" }, { "path": "tests/bin/action_runner_test.py", "content": "import tempfile\nfrom unittest import mock\n\nimport pytest\n\nfrom testifycompat import assert_equal\nfrom testifycompat import setup\nfrom testifycompat import setup_teardown\nfrom testifycompat import TestCase\nfrom tron.bin import action_runner\n\n\nclass TestStatusFile(TestCase):\n @setup\n def setup_status_file(self):\n self.filename = tempfile.NamedTemporaryFile().name\n self.status_file = action_runner.StatusFile(self.filename)\n\n def test_get_content(self):\n command, proc, run_id = \"do this\", mock.Mock(), \"Job.test.1\"\n with (\n mock.patch(\"tron.bin.action_runner.time.time\", autospec=True) as faketime,\n mock.patch(\"tron.bin.action_runner.os.getpid\", autospec=True) as fakepid,\n ):\n faketime.return_value = 0\n fakepid.return_value = 2\n content = self.status_file.get_content(\n command=command,\n proc=proc,\n run_id=run_id,\n )\n expected = dict(\n run_id=run_id,\n command=command,\n pid=proc.pid,\n return_code=proc.returncode,\n runner_pid=2,\n timestamp=0,\n )\n assert_equal(content, expected)\n\n\nclass TestRegister(TestCase):\n mock_isdir = mock_status_file = None\n mock_makedirs = None\n\n @setup_teardown\n def patch_sys(self):\n with (\n mock.patch(\"tron.bin.action_runner.os.path.isdir\", autospec=True) as self.mock_isdir,\n mock.patch(\"tron.bin.action_runner.os.makedirs\", autospec=True) as self.mock_makedirs,\n mock.patch(\"tron.bin.action_runner.os.access\", autospec=True) as self.mock_access,\n mock.patch(\"tron.bin.action_runner.StatusFile\", autospec=True) as self.mock_status_file,\n ):\n self.output_path = \"/bogus/path/does/not/exist\"\n self.command = \"command\"\n self.run_id = \"Job.test.1\"\n self.proc = mock.Mock()\n self.proc.wait.return_value = 0\n yield\n\n def test_validate_output_dir_does_not_exist(self):\n self.mock_isdir.return_value = False\n self.mock_access.return_value = True\n action_runner.validate_output_dir(self.output_path)\n self.mock_makedirs.assert_called_with(self.output_path)\n\n def test_validate_output_dir_does_not_exist_create_fails(self):\n self.mock_isdir.return_value = False\n self.mock_access.return_value = True\n self.mock_makedirs.side_effect = OSError\n with pytest.raises(OSError):\n action_runner.validate_output_dir(self.output_path)\n\n def test_validate_output_dir_exists_not_writable(self):\n self.mock_isdir.return_value = True\n self.mock_access.return_value = False\n with pytest.raises(OSError):\n action_runner.validate_output_dir(self.output_path)\n\n def test_run_proc(self):\n self.mock_isdir.return_value = True\n self.mock_access.return_value = True\n action_runner.run_proc(\n self.output_path,\n self.command,\n self.run_id,\n self.proc,\n )\n self.mock_status_file.assert_called_with(\n self.output_path + \"/\" + action_runner.STATUS_FILE,\n )\n self.mock_status_file.return_value.wrap.assert_called_with(\n command=self.command,\n run_id=self.run_id,\n proc=self.proc,\n )\n self.proc.wait.assert_called_with()\n\n\nclass TestBuildEnvironment:\n def test_build_environment(self):\n with mock.patch(\n \"tron.bin.action_runner.os.environ\",\n dict(PATH=\"/usr/bin/nowhere\"),\n autospec=None,\n ):\n env = action_runner.build_environment(\"MASTER.foo.10.bar\")\n\n assert env == dict(\n PATH=\"/usr/bin/nowhere\",\n TRON_JOB_NAMESPACE=\"MASTER\",\n TRON_JOB_NAME=\"foo\",\n TRON_RUN_NUM=\"10\",\n TRON_ACTION=\"bar\",\n )\n\n def test_build_environment_invalid_run_id(self):\n with mock.patch(\n \"tron.bin.action_runner.os.environ\",\n dict(PATH=\"/usr/bin/nowhere\"),\n autospec=None,\n ):\n env = action_runner.build_environment(\"asdf\")\n\n assert env == dict(\n PATH=\"/usr/bin/nowhere\",\n TRON_JOB_NAMESPACE=\"UNKNOWN\",\n TRON_JOB_NAME=\"UNKNOWN\",\n TRON_RUN_NUM=\"UNKNOWN\",\n TRON_ACTION=\"UNKNOWN\",\n )\n\n def test_build_environment_too_long_run_id(self):\n with mock.patch(\n \"tron.bin.action_runner.os.environ\",\n dict(PATH=\"/usr/bin/nowhere\"),\n autospec=None,\n ):\n env = action_runner.build_environment(\"MASTER.foo.10.bar.baz\")\n\n assert env == dict(\n PATH=\"/usr/bin/nowhere\",\n TRON_JOB_NAMESPACE=\"MASTER\",\n TRON_JOB_NAME=\"foo\",\n TRON_RUN_NUM=\"10\",\n TRON_ACTION=\"bar.baz\",\n )\n\n\nclass TestBuildLabels:\n def test_build_labels(self):\n labels = action_runner.build_labels(\"MASTER.foo.10.bar\")\n\n assert labels == {\n \"tron.yelp.com/run_num\": \"10\",\n }\n\n def test_build_labels_with_merging(self):\n current_labels = {\"LABEL1\": \"value_1\"}\n labels = action_runner.build_labels(\"MASTER.foo.10.bar\", current_labels)\n\n assert labels == {\n \"tron.yelp.com/run_num\": \"10\",\n \"LABEL1\": \"value_1\",\n }\n\n def test_build_labels_with_merging_on_unknown(self):\n current_labels = {\"LABEL1\": \"value_1\"}\n labels = action_runner.build_labels(\"asdf\", current_labels)\n\n assert labels == {\n \"tron.yelp.com/run_num\": \"UNKNOWN\",\n \"LABEL1\": \"value_1\",\n }\n\n def test_build_labels_invalid_run_id(self):\n labels = action_runner.build_labels(\"asdf\")\n\n assert labels == {\n \"tron.yelp.com/run_num\": \"UNKNOWN\",\n }\n\n def test_build_labels_too_long_run_id(self):\n labels = action_runner.build_labels(\"MASTER.foo.10.bar.baz\")\n\n assert labels == {\n \"tron.yelp.com/run_num\": \"10\",\n }\n\n def test_build_labels_with_attempt_number_zero(self):\n labels = action_runner.build_labels(\"MASTER.foo.10.bar\", attempt_number=0)\n\n assert labels == {\n \"tron.yelp.com/run_num\": \"10\",\n \"tron.yelp.com/attempt_number\": \"0\",\n }\n\n def test_build_labels_with_attempt_number_retry(self):\n labels = action_runner.build_labels(\"MASTER.foo.10.bar\", attempt_number=2)\n\n assert labels == {\n \"tron.yelp.com/run_num\": \"10\",\n \"tron.yelp.com/attempt_number\": \"2\",\n }\n\n def test_build_labels_with_attempt_number_and_original_labels(self):\n current_labels = {\"LABEL1\": \"value_1\"}\n labels = action_runner.build_labels(\"MASTER.foo.10.bar\", current_labels, attempt_number=1)\n\n assert labels == {\n \"tron.yelp.com/run_num\": \"10\",\n \"tron.yelp.com/attempt_number\": \"1\",\n \"LABEL1\": \"value_1\",\n }\n\n def test_build_labels_without_attempt_number_omits_label(self):\n labels = action_runner.build_labels(\"MASTER.foo.10.bar\")\n\n assert \"tron.yelp.com/attempt_number\" not in labels\n" }, { "path": "tests/bin/action_status_test.py", "content": "import signal\nimport tempfile\nfrom unittest import mock\n\nfrom testifycompat import setup_teardown\nfrom testifycompat import TestCase\nfrom tron import yaml\nfrom tron.bin import action_status\n\n\nclass TestActionStatus(TestCase):\n @setup_teardown\n def setup_status_file(self):\n self.status_file = tempfile.NamedTemporaryFile(mode=\"r+\")\n self.status_content = {\n \"pid\": 1234,\n \"return_code\": None,\n \"run_id\": \"MASTER.foo.bar.1234\",\n }\n self.status_file.write(yaml.safe_dump(self.status_content))\n self.status_file.flush()\n self.status_file.seek(0)\n yield\n self.status_file.close()\n\n @mock.patch(\"tron.bin.action_status.os.killpg\", autospec=True)\n @mock.patch(\n \"tron.bin.action_status.os.getpgid\",\n autospec=True,\n return_value=42,\n )\n def test_send_signal(self, mock_getpgid, mock_kill):\n action_status.send_signal(signal.SIGKILL, self.status_file)\n mock_getpgid.assert_called_with(self.status_content[\"pid\"])\n mock_kill.assert_called_with(42, signal.SIGKILL)\n\n def test_get_field_retrieves_last_entry(self):\n self.status_file.seek(0, 2)\n additional_status_content = {\n \"pid\": 1234,\n \"return_code\": 0,\n \"run_id\": \"MASTER.foo.bar.1234\",\n \"command\": \"echo \" + \"really_long\" * 100,\n }\n self.status_file.write(\n yaml.safe_dump(additional_status_content, explicit_start=True),\n )\n self.status_file.flush()\n self.status_file.seek(0)\n assert action_status.get_field(\"return_code\", self.status_file) == 0\n\n def test_get_field_none(self):\n assert action_status.get_field(\"return_code\", self.status_file) is None\n" }, { "path": "tests/bin/check_tron_jobs_test.py", "content": "import time\nfrom unittest import mock\nfrom unittest.mock import patch\nfrom unittest.mock import PropertyMock\n\nimport pytest\n\nfrom testifycompat import assert_equal\nfrom testifycompat import TestCase\nfrom tron.bin import check_tron_jobs\nfrom tron.bin.check_tron_jobs import State\n\n\n@pytest.fixture(autouse=True)\ndef mock_run_interval():\n with patch.object(check_tron_jobs, \"_run_interval\", 300):\n yield\n\n\nclass TestCheckJobs(TestCase):\n @patch(\"tron.bin.check_tron_jobs.check_job_result\", autospec=True)\n @patch(\"tron.bin.check_tron_jobs.Client\", autospec=True)\n @patch(\"tron.bin.check_tron_jobs.cmd_utils\", autospec=True)\n @patch(\"tron.bin.check_tron_jobs.parse_cli\", autospec=True)\n def test_check_job_result_exception(\n self,\n mock_args,\n mock_cmd_utils,\n mock_client,\n mock_check_job_result,\n ):\n type(mock_args.return_value).job = PropertyMock(return_value=None)\n mock_client.return_value.jobs.return_value = [\n {\n \"name\": \"job1\",\n },\n {\n \"name\": \"job2\",\n },\n {\n \"name\": \"job3\",\n },\n ]\n mock_check_job_result.side_effect = [\n KeyError(\"foo\"),\n None,\n TypeError,\n ]\n error_code = check_tron_jobs.main()\n assert_equal(error_code, 1)\n assert_equal(mock_check_job_result.call_count, 3)\n\n # These tests test job run succeeded scenarios\n def test_job_succeeded(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"running\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1700),\n ),\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.1\")\n assert_equal(state, State.SUCCEEDED)\n\n def test_job_running_and_action_succeeded(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"running\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.2.action2\",\n \"state\": \"running\",\n },\n {\n \"id\": \"MASTER.test.1.action1\",\n \"state\": \"succeeded\",\n },\n ], # noqa: E122\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1700),\n ),\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.1\")\n assert_equal(state, State.SUCCEEDED)\n\n def test_get_relevant_action_picks_the_first_one_succeeded(self):\n action_runs = [\n {\n \"id\": \"MASTER.test.action1\",\n \"action_name\": \"action1\",\n \"state\": \"succeeded\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1200),\n ),\n \"duration\": \"0:18:01.475067\",\n },\n {\n \"id\": \"MASTER.test.action2\",\n \"action_name\": \"action2\",\n \"state\": \"succeeded\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"duration\": \"0:08:02.005783\",\n },\n {\n \"id\": \"MASTER.test.action1\",\n \"action_name\": \"action1\",\n \"state\": \"succeeded\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time()),\n ),\n \"duration\": \"0:00:01.006305\",\n },\n ]\n actual = check_tron_jobs.get_relevant_action(\n action_runs=action_runs,\n last_state=State.SUCCEEDED,\n actions_expected_runtime={\n \"action1\": 86400.0,\n \"action2\": 86400.0,\n \"action3\": 86400.0,\n },\n )\n assert_equal(actual[\"id\"], \"MASTER.test.action1\")\n\n # These tests test job run failed scenarios\n def test_job_failed(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"running\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"failed\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1700),\n ),\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.1\")\n assert_equal(state, State.FAILED)\n\n def test_most_recent_end_time_job_failed(self):\n job_runs = {\n \"status\": \"scheduled\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"succeeded\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1700),\n ),\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"failed\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 500),\n ),\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.1\")\n assert_equal(state, State.FAILED)\n\n def test_rerun_job_failed(self):\n job_runs = {\n \"status\": \"scheduled\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.4\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"failed\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 100),\n ),\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"succeeded\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 500),\n ),\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"failed\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1700),\n ),\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.3\")\n assert_equal(state, State.FAILED)\n\n def test_job_running_but_action_failed_already(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"running\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.2.action2\",\n \"state\": \"running\",\n },\n {\n \"id\": \"MASTER.test.1.action1\",\n \"state\": \"failed\",\n },\n ], # noqa: E122\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1700),\n ),\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.2\")\n assert_equal(state, State.FAILED)\n\n def test_get_relevant_action_picks_the_one_that_failed(self):\n action_runs = [\n {\n \"node\": {\n \"username\": \"batch\",\n \"hostname\": \"localhost\",\n \"name\": \"localhost\",\n \"port\": 22,\n },\n \"raw_command\": \"/bin/false\",\n \"requirements\": [],\n \"run_num\": \"582\",\n \"exit_status\": 1,\n \"stdout\": None,\n \"start_time\": \"2018-02-05 17:40:00\",\n \"id\": \"MASTER.kwatest.582.action1\",\n \"action_name\": \"action1\",\n \"state\": \"failed\",\n \"command\": \"/bin/false\",\n \"end_time\": \"2018-02-05 17:40:00\",\n \"stderr\": None,\n \"duration\": \"0:00:00.065018\",\n \"job_name\": \"MASTER.kwatest\",\n },\n {\n \"node\": {\n \"username\": \"batch\",\n \"hostname\": \"localhost\",\n \"name\": \"localhost\",\n \"port\": 22,\n },\n \"raw_command\": \"/bin/true\",\n \"requirements\": [],\n \"run_num\": \"582\",\n \"exit_status\": 0,\n \"stdout\": None,\n \"start_time\": \"2018-02-05 17:40:00\",\n \"id\": \"MASTER.kwatest.582.action2\",\n \"action_name\": \"action2\",\n \"state\": \"succeeded\",\n \"command\": \"/bin/true\",\n \"end_time\": \"2018-02-05 17:40:00\",\n \"stderr\": None,\n \"duration\": \"0:00:00.046243\",\n \"job_name\": \"MASTER.kwatest\",\n },\n ]\n actual = check_tron_jobs.get_relevant_action(\n action_runs=action_runs,\n last_state=State.FAILED,\n actions_expected_runtime={},\n )\n assert_equal(actual[\"state\"], \"failed\")\n\n # These tests test job/action stuck scenarios\n def test_job_next_run_starting_no_overlap_is_stuck(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"queued\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"running\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1200),\n ),\n \"end_time\": None,\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.1\")\n assert_equal(state, State.STUCK)\n\n def test_job_next_run_starting_overlap_allowed_not_stuck(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"allow_overlap\": True,\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"queued\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"running\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1200),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1700),\n ),\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.1\")\n assert_equal(state, State.SUCCEEDED)\n\n def test_job_next_run_running_no_queueing_not_stuck(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"allow_overlap\": False,\n \"queueing\": False,\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"cancelled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"running\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1200),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1700),\n ),\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.1\")\n assert_equal(state, State.SUCCEEDED)\n\n def test_job_next_run_starting_no_queueing_not_stuck(self):\n job_runs = {\n \"status\": \"starting\",\n \"next_run\": None,\n \"allow_overlap\": False,\n \"queueing\": False,\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"cancelled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"starting\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1200),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1700),\n ),\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.1\")\n assert_equal(state, State.SUCCEEDED)\n\n def test_job_running_job_exceeds_expected_runtime(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"expected_runtime\": 480.0,\n \"allow_overlap\": True,\n \"runs\": [\n {\n \"id\": \"MASTER.test.100\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"end_time\": None,\n \"start_time\": None,\n \"duration\": \"\",\n },\n {\n \"id\": \"MASTER.test.99\",\n \"state\": \"running\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n \"duration\": \"0:10:01.883601\",\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.99\")\n assert_equal(state, State.STUCK)\n\n def test_job_starting_job_exceeds_expected_runtime(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"expected_runtime\": 480.0,\n \"allow_overlap\": True,\n \"runs\": [\n {\n \"id\": \"MASTER.test.100\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"end_time\": None,\n \"start_time\": None,\n \"duration\": \"\",\n },\n {\n \"id\": \"MASTER.test.99\",\n \"state\": \"starting\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n \"duration\": \"0:10:01.883601\",\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.99\")\n assert_equal(state, State.STUCK)\n\n def test_job_waiting_job_exceeds_expected_runtime_already_started(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"expected_runtime\": 480.0,\n \"allow_overlap\": True,\n \"runs\": [\n {\n \"id\": \"MASTER.test.100\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"end_time\": None,\n \"start_time\": None,\n \"duration\": \"\",\n },\n {\n \"id\": \"MASTER.test.99\",\n \"state\": \"waiting\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n \"duration\": \"0:10:01.883601\",\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.99\")\n assert_equal(state, State.STUCK)\n\n def test_job_running_action_exceeds_expected_runtime(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"actions_expected_runtime\": {\n \"action1\": 720.0,\n \"action2\": 480.0,\n },\n \"runs\": [\n dict(\n id=\"MASTER.test.3\",\n state=\"scheduled\",\n run_time=time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n end_time=None,\n duration=\"\",\n ),\n dict(\n id=\"MASTER.test.2\",\n state=\"running\",\n run_time=time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n end_time=None,\n duration=\"0:10:01.883601\",\n runs=[\n dict(\n id=\"MASTER.test.2.action2\",\n state=\"running\",\n action_name=\"action2\",\n start_time=time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n duration=\"0:10:01.883601\",\n ),\n dict(\n id=\"MASTER.test.2.action1\",\n state=\"running\",\n action_name=\"action1\",\n start_time=time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n duration=\"0:10:01.885401\",\n ),\n ],\n ),\n dict(\n id=\"MASTER.test.1\",\n state=\"succeeded\",\n run_time=time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n end_time=time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1700),\n ),\n duration=\"0:15:00.453601\",\n ),\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.2\")\n assert_equal(state, State.STUCK)\n\n def test_job_running_action_exceeds_expected_runtime_and_other_action_failed(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"actions_expected_runtime\": {\n \"action1\": 720.0,\n \"action2\": 480.0,\n },\n \"runs\": [\n dict(\n id=\"MASTER.test.1\",\n state=\"running\",\n run_time=time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n end_time=None,\n duration=\"0:10:01.883601\",\n runs=[\n dict(\n id=\"MASTER.test.1.action2\",\n state=\"failed\",\n action_name=\"action2\",\n start_time=time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n duration=\"0:10:01.883601\",\n ),\n dict(\n id=\"MASTER.test.1.action1\",\n state=\"running\",\n action_name=\"action1\",\n start_time=time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n duration=\"0:10:01.885401\",\n ),\n ],\n ),\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.1\")\n assert_equal(state, State.FAILED)\n\n def test_job_stuck_when_runtime_not_sorted(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"running\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time()),\n ),\n \"end_time\": None,\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.2\")\n assert_equal(state, State.STUCK)\n\n def test_get_relevant_action_pick_the_one_stuck(self):\n action_runs = [\n {\n \"id\": \"MASTER.test.1.action3\",\n \"state\": \"succeeded\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1200),\n ),\n \"duration\": \"0:18:01.475067\",\n },\n {\n \"id\": \"MASTER.test.1.action2\",\n \"state\": \"running\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1100),\n ),\n \"duration\": \"0:18:40.005783\",\n },\n {\n \"id\": \"MASTER.test.1.action1\",\n \"state\": \"succeeded\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1000),\n ),\n \"duration\": \"0:00:01.006305\",\n },\n ]\n actual = check_tron_jobs.get_relevant_action(\n action_runs=action_runs,\n last_state=State.STUCK,\n actions_expected_runtime={\n \"action1\": 86400.0,\n \"action2\": 86400.0,\n \"action3\": 86400.0,\n },\n )\n assert_equal(actual[\"id\"], \"MASTER.test.1.action2\")\n\n def test_get_relevant_action_pick_the_one_exceeds_expected_runtime(self):\n action_runs = [\n {\n \"id\": \"MASTER.test.1.action3\",\n \"state\": \"running\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"duration\": \"0:10:00.006305\",\n },\n {\n \"id\": \"MASTER.test.1.action2\",\n \"state\": \"running\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"duration\": \"0:10:00.006383\",\n },\n {\n \"id\": \"MASTER.test.1.action1\",\n \"state\": \"succeeded\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"duration\": \"0:10:00.006331\",\n },\n ]\n actions_expected_runtime = {\n \"action3\": 480.0,\n \"action2\": 720.0,\n \"action1\": 900.0,\n }\n actual = check_tron_jobs.get_relevant_action(\n action_runs=action_runs,\n last_state=State.STUCK,\n actions_expected_runtime=actions_expected_runtime,\n )\n assert_equal(actual[\"id\"], \"MASTER.test.1.action3\")\n\n def test_get_relevant_action_pick_the_one_starting(self):\n action_runs = [\n {\n \"id\": \"MASTER.test.1.action3\",\n \"state\": \"starting\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"duration\": \"0:10:00.006305\",\n },\n {\n \"id\": \"MASTER.test.1.action2\",\n \"state\": \"running\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"duration\": \"0:10:00.006383\",\n },\n {\n \"id\": \"MASTER.test.1.action1\",\n \"state\": \"succeeded\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"duration\": \"0:10:00.006331\",\n },\n ]\n actions_expected_runtime = {\n \"action3\": 480.0,\n \"action2\": 720.0,\n \"action1\": 900.0,\n }\n actual = check_tron_jobs.get_relevant_action(\n action_runs=action_runs,\n last_state=State.STUCK,\n actions_expected_runtime=actions_expected_runtime,\n )\n assert_equal(actual[\"id\"], \"MASTER.test.1.action3\")\n\n def test_get_relevant_action_pick_the_one_exceeds_expected_runtime_with_long_duration(\n self,\n ):\n action_runs = [\n {\n \"id\": \"MASTER.test.1.action3\",\n \"action_name\": \"action3\",\n \"state\": \"running\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"duration\": \"1 day, 0:10:00.006305\",\n },\n {\n \"id\": \"MASTER.test.1.action2\",\n \"action_name\": \"action2\",\n \"state\": \"running\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"duration\": \"2 days, 0:10:00.006383\",\n },\n {\n \"id\": \"MASTER.test.1.action1\",\n \"action_name\": \"action1\",\n \"state\": \"running\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"duration\": \"1 day, 0:10:00.006331\",\n },\n ]\n actions_expected_runtime = {\n \"action3\": 100000.0,\n \"action2\": 100000.0,\n \"action1\": 100000.0,\n }\n actual = check_tron_jobs.get_relevant_action(\n action_runs=action_runs,\n last_state=State.STUCK,\n actions_expected_runtime=actions_expected_runtime,\n )\n assert_equal(actual[\"id\"], \"MASTER.test.1.action2\")\n\n def test_no_job_scheduled_or_queuing(self):\n \"\"\"If the past 2 runs succeeded but no future job is scheuled,\n we should consider the job to have suceeded.\n \"\"\"\n job_runs = {\n \"status\": \"succeeded\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"succeeded\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 300),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 900),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1200),\n ),\n },\n ],\n \"monitoring\": {},\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.2\")\n assert_equal(state, State.SUCCEEDED)\n\n # These tests test job without succeeded/failed run scenarios\n def test_job_no_runs_to_check(self):\n job_runs = {\n \"status\": \"scheduled\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 1200),\n ),\n \"end_time\": None,\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.1\")\n assert_equal(state, State.NO_RUNS_TO_CHECK)\n\n def test_job_has_no_runs_at_all(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"runs\": [],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run, None)\n assert_equal(state, State.NO_RUN_YET)\n\n # These tests test job/action unknown scenarios\n def test_job_unknown(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"unknown\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1200),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1700),\n ),\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.2\")\n assert_equal(state, State.UNKNOWN)\n\n def test_job_running_but_action_unknown_already(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"running\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.2.action2\",\n \"state\": \"running\",\n },\n {\n \"id\": \"MASTER.test.1.action1\",\n \"state\": \"unknown\",\n },\n ], # noqa: E122\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1700),\n ),\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.2\")\n assert_equal(state, State.UNKNOWN)\n\n def test_job_waiting_but_action_unknown_already(self):\n job_runs = {\n \"status\": \"waiting\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"scheduled\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"end_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"waiting\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"end_time\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.2.action2\",\n \"state\": \"waiting\",\n },\n {\n \"id\": \"MASTER.test.1.action1\",\n \"state\": \"unknown\",\n },\n ], # noqa: E122\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n \"end_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1700),\n ),\n },\n ],\n }\n run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)\n assert_equal(run[\"id\"], \"MASTER.test.2\")\n assert_equal(state, State.UNKNOWN)\n\n # These tests test guess realert feature\n def test_guess_realert_every(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"scheduled\",\n \"start_time\": None,\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"failed\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n },\n ], # noqa: E122\n }\n realert_every = check_tron_jobs.guess_realert_every(job_runs)\n assert_equal(realert_every, 4)\n\n def test_guess_realert_every_no_action_run_starts(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"scheduled\",\n \"start_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"failed\",\n \"start_time\": None,\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 200),\n ),\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n \"run_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n },\n ], # noqa: E122\n }\n realert_every = check_tron_jobs.guess_realert_every(job_runs)\n assert_equal(realert_every, 2)\n\n def test_guess_realert_every_queue_job(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": None,\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"queued\",\n \"start_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"running\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 600),\n ),\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 1800),\n ),\n },\n ],\n }\n realert_every = check_tron_jobs.guess_realert_every(job_runs)\n assert_equal(realert_every, -1)\n\n def test_guess_realert_every_frequent_run(self):\n job_runs = {\n \"status\": \"running\",\n \"next_run\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 10),\n ),\n \"runs\": [\n {\n \"id\": \"MASTER.test.3\",\n \"state\": \"scheduled\",\n \"start_time\": None,\n },\n {\n \"id\": \"MASTER.test.2\",\n \"state\": \"failed\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 10),\n ),\n },\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"succeeded\",\n \"start_time\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() - 20),\n ),\n },\n ], # noqa: E122\n }\n realert_every = check_tron_jobs.guess_realert_every(job_runs)\n assert_equal(realert_every, 1)\n\n def test_guess_realert_every_first_time_job(self):\n job_runs = {\n \"status\": \"enabled\",\n \"next_run\": time.strftime(\n \"%Y-%m-%d %H:%M:%S\",\n time.localtime(time.time() + 600),\n ),\n \"runs\": [\n {\n \"id\": \"MASTER.test.1\",\n \"state\": \"scheduled\",\n \"start_time\": None,\n },\n ],\n }\n realert_every = check_tron_jobs.guess_realert_every(job_runs)\n assert_equal(realert_every, -1)\n\n\nclass TestCheckPreciousJobs:\n @pytest.fixture(autouse=True)\n def setup_job(self):\n self.job_name = \"fake_job\"\n self.monitoring = {\n \"team\": \"fake_team\",\n \"notification_email\": \"fake_email\",\n check_tron_jobs.PRECIOUS_JOB_ATTR: True,\n }\n self.runs = [\n {\n \"id\": f\"{self.job_name}.15\",\n \"job_name\": self.job_name,\n \"run_num\": 15,\n \"run_time\": \"2018-10-13 12:00:00\",\n \"start_time\": \"2018-10-13 12:00:00\",\n \"end_time\": \"2018-10-13 12:30:00\",\n \"state\": \"succeeded\",\n \"exit_status\": 0,\n },\n {\n \"id\": f\"{self.job_name}.14\",\n \"job_name\": self.job_name,\n \"run_num\": 14,\n \"run_time\": \"2018-10-12 12:00:00\",\n \"start_time\": \"2018-10-12 12:00:00\",\n \"end_time\": \"2018-10-12 12:30:00\",\n \"state\": \"failed\",\n \"exit_status\": 1,\n },\n {\n \"id\": f\"{self.job_name}.13\",\n \"job_name\": self.job_name,\n \"run_num\": 13,\n \"run_time\": \"2018-10-11 13:00:00\",\n \"start_time\": \"2018-10-11 13:00:00\",\n \"end_time\": \"2018-10-11 13:30:00\",\n \"state\": \"failed\",\n \"exit_status\": 1,\n },\n {\n \"id\": f\"{self.job_name}.12\",\n \"job_name\": self.job_name,\n \"run_num\": 12,\n \"run_time\": \"2018-10-11 12:00:00\",\n \"start_time\": \"2018-10-11 12:00:00\",\n \"end_time\": \"2018-10-11 12:30:00\",\n \"state\": \"succeeded\",\n \"exit_status\": 0,\n },\n {\n \"id\": f\"{self.job_name}.11\",\n \"job_name\": self.job_name,\n \"run_num\": 11,\n \"run_time\": \"2018-10-10 13:00:00\",\n \"start_time\": \"2018-10-10 13:00:00\",\n \"end_time\": \"2018-10-10 13:30:00\",\n \"state\": \"succeeded\",\n \"exit_status\": 0,\n },\n {\n \"id\": f\"{self.job_name}.10\",\n \"job_name\": self.job_name,\n \"run_num\": 10,\n \"run_time\": \"2018-10-10 12:00:00\",\n \"start_time\": \"2018-10-10 12:00:00\",\n \"end_time\": \"2018-10-10 12:30:00\",\n \"state\": \"failed\",\n \"exit_status\": 1,\n },\n ]\n self.job = {\n \"name\": \"fake_job\",\n \"status\": \"enabled\",\n \"monitoring\": self.monitoring,\n \"runs\": self.runs,\n }\n\n @patch(\"time.time\", mock.Mock(return_value=1539460800.0), autospec=None)\n def test_sort_runs_by_interval_day(self):\n run_buckets = check_tron_jobs.sort_runs_by_interval(self.job, \"day\")\n\n assert set(run_buckets.keys()) == {\n \"2018.10.10\",\n \"2018.10.11\",\n \"2018.10.12\",\n \"2018.10.13\",\n }\n assert len(run_buckets[\"2018.10.10\"]) == 2\n assert len(run_buckets[\"2018.10.11\"]) == 2\n assert len(run_buckets[\"2018.10.12\"]) == 1\n assert len(run_buckets[\"2018.10.13\"]) == 1\n\n @patch(\"time.time\", mock.Mock(return_value=1539633600.0), autospec=None)\n def test_sort_runs_by_interval_day_empty_buckets(self):\n self.job[\"runs\"] = [\n {\n \"id\": f\"{self.job_name}.16\",\n \"job_name\": self.job_name,\n \"run_num\": 16,\n \"run_time\": \"2018-10-15 12:00:00\",\n \"start_time\": \"2018-10-15 12:00:00\",\n \"end_time\": \"2018-10-15 12:30:00\",\n \"state\": \"succeeded\",\n \"exit_status\": 0,\n }\n ] + self.job[\"runs\"]\n\n run_buckets = check_tron_jobs.sort_runs_by_interval(self.job, \"day\")\n\n assert \"2018.10.14\" in run_buckets\n assert run_buckets[\"2018.10.14\"] == []\n\n @patch(\"time.time\", mock.Mock(return_value=1539633600.0), autospec=None)\n def test_sort_runs_by_interval_day_old_empty_buckets(self):\n # If the newest run is a backfill for an older date, shouldn't be\n # included for buckets\n self.job[\"runs\"] = [\n {\n \"id\": f\"{self.job_name}.16\",\n \"job_name\": self.job_name,\n \"run_num\": 16,\n \"run_time\": \"2018-10-01 12:00:00\",\n \"start_time\": \"2018-10-01 12:00:00\",\n \"end_time\": \"2018-10-01 12:30:00\",\n \"state\": \"succeeded\",\n \"exit_status\": 0,\n }\n ] + self.job[\"runs\"]\n\n run_buckets = check_tron_jobs.sort_runs_by_interval(self.job, \"day\")\n # Current time is patched to 2018-10-15, so we should include\n # NUM_PRECIOUS intervals, starting 2018-10-09\n # 2018-10-01 should not be included, even though there is a run for it,\n # because it is too old\n assert \"2018.10.01\" not in run_buckets\n assert run_buckets[\"2018.10.14\"] == []\n assert len(run_buckets) == 7\n\n @patch(\n \"tron.bin.check_tron_jobs.guess_realert_every\",\n mock.Mock(return_value=1),\n autospec=None,\n )\n @patch(\"tron.bin.check_tron_jobs.Client\", autospec=True)\n @patch(\"tron.bin.check_tron_jobs.compute_check_result_for_job_runs\", autospec=True)\n @patch(\"tron.bin.check_tron_jobs.get_object_type_from_identifier\", autospec=True)\n def test_compute_check_result_for_job_not_precious(\n self,\n mock_get_obj_type,\n mock_check_job_runs,\n mock_client,\n ):\n client = mock_client(\"fake_server\")\n client.cluster_name = \"fake_cluster\"\n del self.job[\"monitoring\"][check_tron_jobs.PRECIOUS_JOB_ATTR]\n client.job = mock.Mock(return_value=self.job)\n mock_check_job_runs.return_value = {\n \"output\": \"fake_output\",\n \"status\": \"fake_status\",\n }\n\n results = check_tron_jobs.compute_check_result_for_job(\n client,\n self.job,\n url_index={},\n )\n\n # make sure all job runs for a job are included by not incl count arg\n assert client.job.call_args_list == [\n mock.call(\n mock_get_obj_type.return_value.url,\n include_action_runs=True,\n ),\n ]\n assert len(results) == 1\n assert results[0][\"name\"] == \"check_tron_job.fake_job\"\n assert mock_check_job_runs.call_count == 1\n\n @patch(\n \"tron.bin.check_tron_jobs.guess_realert_every\",\n mock.Mock(return_value=1),\n autospec=None,\n )\n @patch(\"tron.bin.check_tron_jobs.Client\", autospec=True)\n def test_compute_check_result_for_job_disabled(self, mock_client):\n client = mock_client(\"fake_server\")\n client.cluster_name = \"fake_cluster\"\n self.job[\"status\"] = \"disabled\"\n\n results = check_tron_jobs.compute_check_result_for_job(\n client,\n self.job,\n url_index={},\n )\n\n assert len(results) == 1\n assert results[0][\"status\"] == 0\n assert results[0][\"output\"] == \"OK: fake_job is disabled and won't be checked.\"\n\n @patch(\n \"tron.bin.check_tron_jobs.guess_realert_every\",\n mock.Mock(return_value=1),\n autospec=None,\n )\n @patch(\"time.time\", mock.Mock(return_value=1539460800.0), autospec=None)\n @patch(\"tron.bin.check_tron_jobs.Client\", autospec=True)\n @patch(\"tron.bin.check_tron_jobs.compute_check_result_for_job_runs\", autospec=True)\n @patch(\"tron.bin.check_tron_jobs.get_object_type_from_identifier\", autospec=True)\n def test_compute_check_result_for_job_enabled(\n self,\n mock_get_obj_type,\n mock_check_job_runs,\n mock_client,\n ):\n client = mock_client(\"fake_server\")\n client.cluster_name = \"fake_cluster\"\n self.job[\"monitoring\"][\"check_every\"] = 500\n self.job[\"monitoring\"][\"hide_stderr\"] = True\n client.job = mock.Mock(return_value=self.job)\n mock_check_job_runs.return_value = {\n \"output\": \"fake_output\",\n \"status\": \"fake_status\",\n }\n\n results = check_tron_jobs.compute_check_result_for_job(\n client,\n self.job,\n url_index={},\n )\n\n # Test that hide_stderr is passed to check_job_runs\n assert mock_check_job_runs.call_args_list[0][1][\"hide_stderr\"] is True\n\n # make sure all job runs for a job are included by not incl count arg\n assert client.job.call_args_list == [\n mock.call(\n mock_get_obj_type.return_value.url,\n include_action_runs=True,\n ),\n ]\n assert len(results) == 4\n assert {res[\"name\"] for res in results} == {\n \"check_tron_job.fake_job-2018.10.10\",\n \"check_tron_job.fake_job-2018.10.11\",\n \"check_tron_job.fake_job-2018.10.12\",\n \"check_tron_job.fake_job-2018.10.13\",\n }\n for res in results:\n assert res[\"check_every\"] == \"300s\"\n assert check_tron_jobs.PRECIOUS_JOB_ATTR not in res\n" }, { "path": "tests/bin/get_tron_metrics_test.py", "content": "import subprocess\nfrom unittest import mock\n\nimport pytest\n\nfrom tron.bin import get_tron_metrics\n\n\ndef test_send_data_metric():\n process = mock.Mock()\n process.communicate = mock.Mock(return_value=(b\"fake_output\", b\"fake_error\"))\n cmd_str = \"meteorite data -v fake_name fake_metric_type fake_value \" \"-d fake_dim_key:fake_dim_value\"\n\n with mock.patch(\n \"subprocess.Popen\",\n mock.Mock(return_value=process),\n autospec=None,\n ) as mock_popen:\n get_tron_metrics.send_data_metric(\n name=\"fake_name\",\n metric_type=\"fake_metric_type\",\n value=\"fake_value\",\n dimensions={\"fake_dim_key\": \"fake_dim_value\"},\n dry_run=False,\n )\n\n assert mock_popen.call_count == 1\n assert mock_popen.call_args == mock.call(\n cmd_str.split(),\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n )\n\n\ndef test_send_data_metric_dry_run():\n with mock.patch(\"subprocess.Popen\", autospec=True) as mock_popen:\n get_tron_metrics.send_data_metric(\n name=\"fake_name\",\n metric_type=\"fake_metric_type\",\n value=\"fake_value\",\n dimensions=\"fake_dimensions\",\n dry_run=True,\n )\n\n assert mock_popen.call_count == 0\n\n\n@mock.patch(\"tron.bin.get_tron_metrics.send_data_metric\", autospec=True)\ndef test_send_counter(mock_send_data_metric):\n kwargs = dict(count=\"fake_count\")\n\n get_tron_metrics.send_counter(\"fake_name\", **kwargs)\n\n assert mock_send_data_metric.call_count == 1\n assert mock_send_data_metric.call_args == mock.call(\n name=\"fake_name\",\n metric_type=\"counter\",\n value=\"fake_count\",\n dimensions={},\n dry_run=False,\n )\n\n\n@mock.patch(\"tron.bin.get_tron_metrics.send_data_metric\", autospec=True)\ndef test_send_gauge(mock_send_data_metric):\n kwargs = dict(value=\"fake_value\")\n\n get_tron_metrics.send_gauge(\"fake_name\", **kwargs)\n\n assert mock_send_data_metric.call_count == 1\n assert mock_send_data_metric.call_args == mock.call(\n name=\"fake_name\",\n metric_type=\"gauge\",\n value=\"fake_value\",\n dimensions={},\n dry_run=False,\n )\n\n\n@mock.patch(\"tron.bin.get_tron_metrics.send_counter\", autospec=True)\ndef test_send_meter(mock_send_counter):\n get_tron_metrics.send_meter(\"fake_name\")\n\n assert mock_send_counter.call_count == 1\n assert mock_send_counter.call_args == mock.call(\"fake_name\")\n\n\n@mock.patch(\"tron.bin.get_tron_metrics.send_gauge\", autospec=True)\ndef test_send_histogram(mock_send_gauge):\n kwargs = dict(\n p50=\"fake_p50\",\n p75=\"fake_p75\",\n p95=\"fake_p95\",\n p99=\"fake_p99\",\n )\n p50_kwargs = dict(\n **kwargs,\n value=\"fake_p50\",\n )\n\n get_tron_metrics.send_histogram(\"fake_name\", **kwargs)\n\n assert mock_send_gauge.call_count == len(kwargs)\n assert mock_send_gauge.call_args_list[0] == mock.call(\n \"fake_name.p50\",\n **p50_kwargs,\n )\n\n\n@mock.patch(\"tron.bin.get_tron_metrics.send_meter\", autospec=True)\n@mock.patch(\"tron.bin.get_tron_metrics.send_histogram\", autospec=True)\ndef test_send_timer(mock_send_meter, mock_send_histogram):\n get_tron_metrics.send_timer(\"fake_name\")\n\n assert mock_send_meter.call_count == 1\n assert mock_send_meter.call_args == mock.call(\"fake_name\")\n assert mock_send_histogram.call_count == 1\n assert mock_send_histogram.call_args == mock.call(\"fake_name\")\n\n\n@pytest.mark.parametrize(\"cluster\", [\"fake_cluster\", None])\ndef test_send_metrics(cluster):\n mock_send_counter = mock.Mock()\n metrics = dict(counter=[dict(name=\"fake_name\")])\n\n with mock.patch(\n \"tron.bin.get_tron_metrics._METRIC_SENDERS\",\n dict(counter=mock_send_counter),\n autospec=None,\n ):\n get_tron_metrics.send_metrics(metrics, cluster=cluster, dry_run=True)\n\n assert mock_send_counter.call_count == 1\n if cluster:\n assert mock_send_counter.call_args == mock.call(\n \"fake_name\",\n dry_run=True,\n dimensions={\"tron_cluster\": \"fake_cluster\"},\n )\n else:\n assert mock_send_counter.call_args == mock.call(\"fake_name\", dry_run=True)\n" }, { "path": "tests/bin/recover_batch_test.py", "content": "import tempfile\nfrom queue import Queue\nfrom unittest import mock\n\nimport pytest\n\nfrom tron.bin import recover_batch\nfrom tron.bin.action_runner import StatusFile\n\n\n@pytest.fixture\ndef mock_file():\n f = tempfile.NamedTemporaryFile()\n yield f.name\n f.close()\n\n\n@mock.patch.object(recover_batch, \"reactor\")\n@mock.patch(\"tron.bin.recover_batch.get_exit_code\", autospec=True)\n@pytest.mark.parametrize(\n \"exit_code,error_msg,should_stop\",\n [\n (1, \"failed\", True),\n (None, None, False),\n ],\n)\ndef test_notify(mock_get_exit_code, mock_reactor, exit_code, error_msg, should_stop):\n mock_get_exit_code.return_value = exit_code, error_msg\n queue = Queue()\n path = mock.Mock()\n recover_batch.notify(queue, \"some_ignored\", path, \"mask\")\n if should_stop:\n assert mock_reactor.stop.call_count == 1\n assert queue.get_nowait() == (exit_code, error_msg)\n else:\n assert mock_reactor.stop.call_count == 0\n assert queue.empty()\n\n\n@mock.patch(\"tron.bin.recover_batch.psutil.pid_exists\", autospec=True)\n@mock.patch(\"tron.bin.recover_batch.read_last_yaml_entries\", autospec=True)\n@pytest.mark.parametrize(\n \"line,exit_code,is_running,error_msg\",\n [\n (\n {\"return_code\": 0, \"runner_pid\": 12345},\n 0,\n False,\n None,\n ), # action runner finishes successfully\n ( # action runner is killed\n {\"return_code\": -9, \"runner_pid\": 12345},\n 9,\n False,\n \"Action run killed by signal SIGKILL\",\n ),\n ( # No return code but action_runner pid is not running\n {\"runner_pid\": 12345},\n 1,\n False,\n \"Action runner pid 12345 no longer running. Assuming an exit of 1.\",\n ),\n (\n {\"runner_pid\": 12345},\n None,\n True,\n None,\n ), # No return code but action_runner pid is running\n (\n {},\n None,\n Exception,\n None,\n ), # No return code or PID from the file\n ],\n)\ndef test_get_exit_code(\n mock_read_last_yaml_entries,\n mock_pid_running,\n line,\n exit_code,\n is_running,\n error_msg,\n):\n fake_path = \"/file/path\"\n mock_read_last_yaml_entries.return_value = line\n mock_pid_running.side_effect = [is_running]\n\n actual_exit_code, actual_error_msg = recover_batch.get_exit_code(fake_path)\n assert actual_exit_code == exit_code\n assert actual_error_msg == error_msg\n\n\ndef test_read_last_yaml_roundtrip(mock_file):\n \"\"\"Check that read_last_yaml_entries returns the same thing that the action\n runner wrote.\"\"\"\n status = StatusFile(mock_file)\n expected_content = [\n {\"return_code\": None, \"pid\": 10345, \"command\": \"foo\"},\n {\"return_code\": 1, \"pid\": 10345, \"command\": \"foo\"},\n ]\n with mock.patch.object(status, \"get_content\", side_effect=expected_content):\n with status.wrap(command=\"echo hello\", run_id=\"job.1.action\", proc=mock.Mock()):\n # In the context manager, we've written the first value of get_content.\n first = recover_batch.read_last_yaml_entries(mock_file)\n assert first == expected_content[0]\n\n # After, we write another status entry. We should return the latest.\n second = recover_batch.read_last_yaml_entries(mock_file)\n assert second == expected_content[1]\n\n\n@mock.patch.object(recover_batch, \"reactor\")\n@mock.patch(\"tron.bin.recover_batch.Queue\", autospec=True)\n@mock.patch(\"tron.bin.recover_batch.get_exit_code\", autospec=True, return_value=(None, None))\n@mock.patch(\"tron.bin.recover_batch.StatusFileWatcher\", autospec=True)\n@pytest.mark.parametrize(\"existing_code,watcher_code\", [(None, 1), (123, None)])\ndef test_run(\n mock_watcher,\n mock_get_exit_code,\n mock_queue,\n mock_reactor,\n existing_code,\n watcher_code,\n):\n mock_get_exit_code.return_value = (existing_code, \"\")\n mock_queue.return_value.get.return_value = (watcher_code, \"\")\n mock_path = mock.Mock()\n if existing_code is not None:\n expected = existing_code\n else:\n expected = watcher_code\n\n with pytest.raises(SystemExit) as e:\n recover_batch.run(mock_path)\n assert e.code == expected\n\n assert mock_get_exit_code.call_args_list == [mock.call(mock_path)]\n if existing_code is not None:\n assert mock_watcher.call_count == 0\n else:\n assert mock_watcher.call_count == 1\n" }, { "path": "tests/command_context_test.py", "content": "import datetime\nfrom unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_raises\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tron import command_context\nfrom tron import node\nfrom tron import scheduler\nfrom tron.core import actionrun\nfrom tron.core import job\nfrom tron.core import jobrun\nfrom tron.core.jobrun import JobRunCollection\n\n\nclass TestEmptyContext(TestCase):\n @setup\n def build_context(self):\n self.context = command_context.CommandContext(None)\n\n def test__getitem__(self):\n assert_raises(KeyError, self.context.__getitem__, \"foo\")\n\n def test_get(self):\n assert not self.context.get(\"foo\")\n\n\nclass TestBuildFilledContext(TestCase):\n def test_build_filled_context_no_objects(self):\n output = command_context.build_filled_context()\n assert not output.base\n assert not output.next\n\n def test_build_filled_context_single(self):\n output = command_context.build_filled_context(\n command_context.JobContext,\n )\n assert isinstance(output.base, command_context.JobContext)\n assert not output.next\n\n def test_build_filled_context_chain(self):\n objs = [command_context.JobContext, command_context.JobRunContext]\n output = command_context.build_filled_context(*objs)\n assert isinstance(output.base, objs[1])\n assert isinstance(output.next.base, objs[0])\n assert not output.next.next\n\n\nclass SimpleContextTestCaseBase(TestCase):\n __test__ = False\n\n def test_hit(self):\n assert_equal(self.context[\"foo\"], \"bar\")\n\n def test_miss(self):\n assert_raises(KeyError, self.context.__getitem__, \"your_mom\")\n\n def test_get_hit(self):\n assert_equal(self.context.get(\"foo\"), \"bar\")\n\n def test_get_miss(self):\n assert not self.context.get(\"unknown\")\n\n\nclass SimpleDictContextTestCase(SimpleContextTestCaseBase):\n @setup\n def build_context(self):\n self.context = command_context.CommandContext(dict(foo=\"bar\"))\n\n\nclass SimpleObjectContextTestCase(SimpleContextTestCaseBase):\n @setup\n def build_context(self):\n class Obj:\n foo = \"bar\"\n\n self.context = command_context.CommandContext(Obj)\n\n\nclass ChainedDictContextTestCase(SimpleContextTestCaseBase):\n @setup\n def build_context(self):\n self.next_context = command_context.CommandContext(\n dict(foo=\"bar\", next_foo=\"next_bar\"),\n )\n self.context = command_context.CommandContext(\n dict(),\n self.next_context,\n )\n\n def test_chain_get(self):\n assert_equal(self.context[\"next_foo\"], \"next_bar\")\n\n\nclass ChainedDictOverrideContextTestCase(SimpleContextTestCaseBase):\n @setup\n def build_context(self):\n self.next_context = command_context.CommandContext(\n dict(foo=\"your mom\", next_foo=\"next_bar\"),\n )\n self.context = command_context.CommandContext(\n dict(foo=\"bar\"),\n self.next_context,\n )\n\n def test_chain_get(self):\n assert_equal(self.context[\"next_foo\"], \"next_bar\")\n\n\nclass ChainedObjectOverrideContextTestCase(SimpleContextTestCaseBase):\n @setup\n def build_context(self):\n class MyObject(TestCase):\n pass\n\n obj = MyObject()\n obj.foo = \"bar\"\n\n self.next_context = command_context.CommandContext(\n dict(foo=\"your mom\", next_foo=\"next_bar\"),\n )\n self.context = command_context.CommandContext(obj, self.next_context)\n\n def test_chain_get(self):\n assert_equal(self.context[\"next_foo\"], \"next_bar\")\n\n\nclass TestJobContext(TestCase):\n @setup\n def setup_job(self):\n self.last_success = mock.Mock(run_time=datetime.datetime(2012, 3, 14))\n mock_scheduler = mock.create_autospec(scheduler.GeneralScheduler)\n run_collection = mock.create_autospec(\n JobRunCollection,\n last_success=self.last_success,\n )\n self.job = job.Job(\n \"MASTER.jobname\",\n mock_scheduler,\n run_collection=run_collection,\n )\n self.context = command_context.JobContext(self.job)\n\n def test_name(self):\n assert_equal(self.context.name, self.job.name)\n\n def test__getitem__last_success(self):\n item = self.context[\"last_success#day-1\"]\n expected_date = self.last_success.run_time - datetime.timedelta(days=1)\n assert_equal(item, str(expected_date.day))\n\n item = self.context[\"last_success#shortdate\"]\n assert_equal(item, \"2012-03-14\")\n\n def test__getitem__last_success_bad_date_spec(self):\n name = \"last_success#beers-3\"\n assert_raises(KeyError, lambda: self.context[name])\n\n def test__getitem__last_success_bad_date_name(self):\n name = \"first_success#shortdate-1\"\n assert_raises(KeyError, lambda: self.context[name])\n\n def test__getitem__last_success_no_date_spec(self):\n name = \"last_success\"\n assert_raises(KeyError, lambda: self.context[name])\n\n def test__getitem__missing(self):\n assert_raises(KeyError, lambda: self.context[\"bogus\"])\n\n def test_namespace(self):\n assert self.context.namespace == \"MASTER\"\n\n\nclass TestJobRunContext(TestCase):\n @setup\n def setup_context(self):\n self.jobrun = mock.create_autospec(jobrun.JobRun, run_time=\"sometime\", manual=True)\n self.context = command_context.JobRunContext(self.jobrun)\n\n def test_cleanup_job_status(self):\n self.jobrun.action_runs.is_failed = False\n self.jobrun.action_runs.is_complete_without_cleanup = True\n assert_equal(self.context.cleanup_job_status, \"SUCCESS\")\n\n def test_cleanup_job_status_failure(self):\n self.jobrun.action_runs.is_failed = True\n assert_equal(self.context.cleanup_job_status, \"FAILURE\")\n\n def test_runid(self):\n assert_equal(self.context.runid, self.jobrun.id)\n\n def test_manual_run(self):\n assert self.context.manual == \"true\"\n\n @mock.patch(\"tron.command_context.timeutils.DateArithmetic\", autospec=True)\n def test__getitem__(self, mock_date_math):\n name = \"date_name\"\n time_value = self.context[name]\n mock_date_math.parse.assert_called_with(name, self.jobrun.run_time)\n assert_equal(time_value, mock_date_math.parse.return_value)\n\n\nclass TestActionRunContext(TestCase):\n @setup\n def build_context(self):\n mock_node = mock.create_autospec(node.Node, hostname=\"something\")\n self.action_run = mock.create_autospec(\n actionrun.ActionRun,\n action_name=\"something\",\n node=mock_node,\n )\n self.context = command_context.ActionRunContext(self.action_run)\n\n def test_actionname(self):\n assert_equal(self.context.actionname, self.action_run.action_name)\n\n def test_node_hostname(self):\n assert_equal(self.context.node, self.action_run.node.hostname)\n\n\nclass TestFiller(TestCase):\n @setup\n def setup_filler(self):\n self.filler = command_context.Filler()\n\n def test_filler_with_job__getitem__(self):\n context = command_context.JobContext(self.filler)\n todays_date = datetime.date.today().strftime(\"%Y-%m-%d\")\n assert_equal(context[\"last_success#shortdate\"], todays_date)\n\n def test_filler_with_job_run__getitem__(self):\n context = command_context.JobRunContext(self.filler)\n todays_date = datetime.date.today().strftime(\"%Y-%m-%d\")\n assert_equal(context[\"shortdate\"], todays_date)\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/commands/__init__.py", "content": "" }, { "path": "tests/commands/backfill_test.py", "content": "import datetime\nfrom unittest import mock\n\nimport pytest\n\nfrom tron.commands import backfill\nfrom tron.commands import client\n\nTEST_DATETIME_1 = datetime.datetime.strptime(\"2004-07-01\", \"%Y-%m-%d\")\nTEST_DATETIME_2 = datetime.datetime.strptime(\"2004-07-02\", \"%Y-%m-%d\")\nTEST_DATETIME_3 = datetime.datetime.strptime(\"2004-07-03\", \"%Y-%m-%d\")\n\n\n@pytest.fixture(autouse=True)\ndef mock_sleep():\n async def empty_coro(*args, **kwargs):\n return None\n\n with mock.patch(\"asyncio.sleep\", empty_coro, autospec=None):\n yield\n\n\n@pytest.fixture(autouse=True)\ndef mock_client():\n with mock.patch.object(client, \"Client\", autospec=True) as m:\n yield m\n\n\n@pytest.fixture(autouse=True)\ndef mock_urlopen(): # prevent any requests from being made\n with mock.patch(\"urllib.request.urlopen\", autospec=True) as m:\n yield m\n\n\n@pytest.fixture\ndef mock_client_request():\n with mock.patch.object(client, \"request\", autospec=True) as m:\n m.return_value = mock.Mock(error=False, content={}) # response\n yield m\n\n\n@pytest.fixture\ndef fake_backfill_run(mock_client):\n tron_client = mock_client.return_value\n tron_client.url_base = \"http://localhost\"\n yield backfill.BackfillRun(\n tron_client,\n client.TronObjectIdentifier(\"JOB\", \"/a_job\"),\n TEST_DATETIME_1,\n )\n\n\n@pytest.mark.parametrize(\n \"is_error,result,expected\",\n [\n (True, \"an_error_msg\", None), # tron api failed\n (False, \"weird_resp_msg\", None), # bad response, can't get job run name\n (False, \"Created JobRun:real_job_run_name\", \"real_job_run_name\"), # ok\n ],\n)\ndef test_backfill_run_create(mock_client_request, fake_backfill_run, event_loop, is_error, result, expected):\n mock_client_request.return_value.error = is_error\n mock_client_request.return_value.content[\"result\"] = result\n assert expected == event_loop.run_until_complete(fake_backfill_run.create())\n\n\n@pytest.mark.parametrize(\n \"obj_type,expected\",\n [\n (client.RequestError(\"\"), None),\n ([client.TronObjectIdentifier(\"JOB_RUN\", \"/a_run\")], client.TronObjectIdentifier(\"JOB_RUN\", \"/a_run\")),\n ],\n)\n@mock.patch.object(client, \"get_object_type_from_identifier\", autospec=True)\ndef test_backfill_run_get_run_id(mock_get_obj_type, fake_backfill_run, event_loop, obj_type, expected):\n mock_get_obj_type.side_effect = obj_type\n assert expected == event_loop.run_until_complete(fake_backfill_run.get_run_id())\n assert expected == fake_backfill_run.run_id\n\n\n@pytest.mark.parametrize(\n \"job_run_resp,expected\",\n [\n (client.RequestError, \"unknown\"), # polling failed\n ([{}], \"unknown\"), # default to unknown\n ([{\"state\": \"failed\"}], \"failed\"), # ok\n ],\n)\ndef test_backfill_run_sync_state(fake_backfill_run, event_loop, job_run_resp, expected):\n fake_backfill_run.run_id = client.TronObjectIdentifier(\"JOB_RUN\", \"/a_run\")\n fake_backfill_run.tron_client.job_runs.side_effect = job_run_resp\n assert expected == event_loop.run_until_complete(fake_backfill_run.sync_state())\n\n\ndef test_backfill_run_watch_until_completion(fake_backfill_run, event_loop):\n async def change_run_state():\n fake_backfill_run.run_state = \"cancelled\"\n\n fake_backfill_run.sync_state = change_run_state\n assert \"cancelled\" == event_loop.run_until_complete(fake_backfill_run.watch_until_completion())\n\n\n@pytest.mark.parametrize(\n \"run_id,response,expected\",\n [\n (None, mock.Mock(error=False), False), # no run_id\n (client.TronObjectIdentifier(\"JOB_RUN\", \"/a_run\"), mock.Mock(error=True), False), # api error\n (client.TronObjectIdentifier(\"JOB_RUN\", \"/a_run\"), mock.Mock(error=False), True), # ok\n ],\n)\ndef test_backfill_run_cancel(\n mock_client_request,\n fake_backfill_run,\n event_loop,\n run_id,\n response,\n expected,\n):\n fake_backfill_run.run_id = run_id\n mock_client_request.return_value = response\n assert expected == event_loop.run_until_complete(fake_backfill_run.cancel())\n\n\n@mock.patch(\"tron.commands.backfill.get_auth_token\", lambda: \"\")\n@mock.patch.object(client, \"get_object_type_from_identifier\", autospec=True)\ndef test_run_backfill_for_date_range_job_dne(mock_get_obj_type, event_loop):\n mock_get_obj_type.side_effect = ValueError\n with pytest.raises(ValueError):\n event_loop.run_until_complete(\n backfill.run_backfill_for_date_range(\"a_server\", \"a_job\", []),\n )\n\n\n@mock.patch(\"tron.commands.backfill.get_auth_token\", lambda: \"\")\n@mock.patch.object(client, \"get_object_type_from_identifier\", autospec=True)\ndef test_run_backfill_for_date_range_not_a_job(mock_get_obj_type, event_loop):\n mock_get_obj_type.return_value = client.TronObjectIdentifier(\"JOB_RUN\", \"a_url\")\n with pytest.raises(ValueError):\n event_loop.run_until_complete(\n backfill.run_backfill_for_date_range(\"a_server\", \"a_job\", []),\n )\n\n\n@pytest.mark.parametrize(\n \"ignore_errors,expected\",\n [\n (True, {\"succeeded\", \"failed\", \"unknown\"}),\n (False, {\"succeeded\", \"failed\", \"not started\"}),\n ],\n)\n@mock.patch(\"tron.commands.backfill.get_auth_token\", lambda: \"\")\n@mock.patch.object(client, \"get_object_type_from_identifier\", autospec=True)\ndef test_run_backfill_for_date_range_normal(mock_get_obj_type, event_loop, ignore_errors, expected):\n run_states = (state for state in [\"succeeded\", \"failed\", \"unknown\"])\n\n async def fake_run_until_completion(self):\n self.run_state = next(run_states)\n\n backfill.BackfillRun.run_until_completion = fake_run_until_completion\n dates = [TEST_DATETIME_1, TEST_DATETIME_2, TEST_DATETIME_3]\n mock_get_obj_type.return_value = client.TronObjectIdentifier(\"JOB\", \"a_url\")\n\n backfill_runs = event_loop.run_until_complete(\n backfill.run_backfill_for_date_range(\n \"a_server\",\n \"a_job\",\n dates,\n max_parallel=2,\n ignore_errors=ignore_errors,\n )\n )\n\n assert {br.run_state for br in backfill_runs} == expected\n" }, { "path": "tests/commands/client_test.py", "content": "from unittest import mock\nfrom urllib.error import HTTPError\nfrom urllib.error import URLError\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_in\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import setup_teardown\nfrom testifycompat import TestCase\nfrom tests.assertions import assert_raises\nfrom tests.testingutils import autospec_method\nfrom tron.commands import client\nfrom tron.commands.client import get_object_type_from_identifier\nfrom tron.commands.client import Response\nfrom tron.commands.client import TronObjectType\n\n\ndef build_file_mock(content):\n return mock.Mock(\n read=mock.Mock(return_value=content),\n headers=mock.Mock(get_content_charset=mock.Mock(return_value=\"utf-8\")),\n )\n\n\nclass TestRequest(TestCase):\n @setup\n def setup_options(self):\n self.url = \"http://localhost:8089/jobs/\"\n\n @setup_teardown\n def patch_urllib(self):\n patcher = mock.patch(\n \"tron.commands.client.urllib.request.urlopen\",\n autospec=True,\n )\n with patcher as self.mock_urlopen:\n yield\n\n def test_build_url_request_no_data(self):\n request = client.build_url_request(self.url, None)\n assert request.has_header(\"User-agent\")\n assert_equal(request.get_method(), \"GET\")\n assert_equal(request.get_full_url(), self.url)\n\n def test_build_url_request_with_data(self):\n data = {\"param\": \"is_set\", \"other\": 1}\n request = client.build_url_request(self.url, data)\n assert request.has_header(\"User-agent\")\n assert_equal(request.get_method(), \"POST\")\n assert_equal(request.get_full_url(), self.url)\n assert_in(\"param=is_set\", request.data.decode())\n assert_in(\"other=1\", request.data.decode())\n\n @mock.patch(\"tron.commands.client.log\", autospec=True)\n def test_load_response_content_success(self, _):\n content = b\"not:valid:json\"\n http_response = build_file_mock(content)\n response = client.load_response_content(http_response)\n assert_equal(response.error, client.DECODE_ERROR)\n assert_equal(response.content, content.decode(\"utf-8\"))\n\n @mock.patch(\"tron.commands.client.log\", autospec=True)\n def test_request_http_error(self, _):\n self.mock_urlopen.side_effect = HTTPError(\n self.url,\n 500,\n \"broke\",\n mock.Mock(get_content_charset=mock.Mock(return_value=\"utf-8\")),\n build_file_mock(b\"oops\"),\n )\n response = client.request(self.url)\n expected = client.Response(500, \"broke\", \"oops\")\n assert_equal(response, expected)\n\n @mock.patch(\"tron.commands.client.log\", autospec=True)\n def test_request_url_error(self, _):\n self.mock_urlopen.side_effect = URLError(\"broke\")\n response = client.request(self.url)\n expected = client.Response(client.URL_ERROR, \"broke\", None)\n assert_equal(response, expected)\n\n def test_request_success(self):\n self.mock_urlopen.return_value = build_file_mock(b'{\"ok\": \"ok\"}')\n response = client.request(self.url)\n expected = client.Response(None, None, {\"ok\": \"ok\"})\n assert_equal(response, expected)\n\n\nclass TestClientRequest(TestCase):\n @setup\n def setup_client(self):\n self.url = \"http://localhost:8089/\"\n self.client = client.Client(self.url)\n\n @setup_teardown\n def patch_request(self):\n with mock.patch(\n \"tron.commands.client.request\",\n autospec=True,\n ) as self.mock_request:\n yield\n\n def test_request_error(self):\n error_response = Response(\n error=\"404\",\n msg=\"Not Found\",\n content=\"big kahuna error\",\n )\n client.request = mock.Mock(return_value=error_response)\n exception = assert_raises(\n client.RequestError,\n self.client.request,\n \"/jobs\",\n )\n\n assert str(exception) == error_response.content\n\n def test_request_success(self):\n ok_response = {\"ok\": \"ok\"}\n client.request.return_value = client.Response(None, None, ok_response)\n response = self.client.request(\"/jobs\")\n assert_equal(response, ok_response)\n\n\nclass TestClient(TestCase):\n @setup\n def setup_client(self):\n self.url = \"http://localhost:8089/\"\n self.client = client.Client(self.url)\n autospec_method(self.client.request)\n\n def test_config_post(self):\n name, data, hash = \"name\", \"stuff\", \"hash\"\n self.client.config(name, config_data=data, config_hash=hash)\n expected_data = {\n \"config\": data,\n \"name\": name,\n \"hash\": hash,\n \"check\": 0,\n }\n self.client.request.assert_called_with(\"/api/config\", expected_data)\n\n def test_config_get_default(self):\n self.client.config(\"config_name\")\n self.client.request.assert_called_with(\n \"/api/config?name=config_name\",\n )\n\n def test_http_get(self):\n self.client.http_get(\"/api/jobs\", {\"include\": 1})\n self.client.request.assert_called_with(\"/api/jobs?include=1\")\n\n def test_action_runs(self):\n self.client.action_runs(\"/api/jobs/name/0/act\", num_lines=40)\n self.client.request.assert_called_with(\n \"/api/jobs/name/0/act?include_stderr=1&include_stdout=1&num_lines=40\",\n )\n\n def test_job_runs(self):\n self.client.job_runs(\"/api/jobs/name/0\")\n self.client.request.assert_called_with(\n \"/api/jobs/name/0?include_action_graph=0&include_action_runs=1\",\n )\n\n def test_job(self):\n self.client.job(\"/api/jobs/name\", count=20)\n self.client.request.assert_called_with(\n \"/api/jobs/name?include_action_runs=0&num_runs=20\",\n )\n\n def test_jobs(self):\n self.client.jobs()\n self.client.request.assert_called_with(\n \"/api/jobs?include_action_graph=1&include_action_runs=0&include_job_runs=0&include_node_pool=1\",\n )\n\n\nclass TestUserAttribution(TestCase):\n def test_default_user_agent(self):\n url = \"http://localhost:8089/\"\n with mock.patch(\n \"tron.commands.client.os.environ\",\n autospec=True,\n ) as mock_environ:\n mock_environ.get.return_value = \"testuser\"\n default_client = client.Client(url, user_attribution=False)\n # we do not add user attribution by default\n assert \"(testuser)\" not in default_client.headers[\"User-Agent\"]\n\n def test_attributed_user_agent(self):\n url = \"http://localhost:8089/\"\n with mock.patch(\n \"tron.commands.client.os.environ\",\n autospec=True,\n ) as mock_environ:\n mock_environ.get.return_value = \"testuser\"\n default_client = client.Client(url, user_attribution=True)\n # we do not add user attribution by default\n assert \"(testuser)\" in default_client.headers[\"User-Agent\"]\n\n\nclass TestGetUrl(TestCase):\n def test_get_job_url_for_action_run(self):\n url = client.get_job_url(\"MASTER.name.1.act\")\n assert_equal(url, \"/api/jobs/MASTER.name/1/act\")\n\n def test_get_job_url_for_job(self):\n url = client.get_job_url(\"MASTER.name\")\n assert_equal(url, \"/api/jobs/MASTER.name\")\n\n\nclass TestGetContentFromIdentifier(TestCase):\n @setup\n def setup_client(self):\n self.options = mock.Mock()\n self.index = {\n \"namespaces\": [\"OTHER\", \"MASTER\"],\n \"jobs\": {\n \"MASTER.namea\": \"\",\n \"MASTER.nameb\": \"\",\n \"OTHER.nameg\": \"\",\n },\n }\n\n def test_get_url_from_identifier_job_no_namespace(self):\n identifier = get_object_type_from_identifier(self.index, \"namea\")\n assert_equal(identifier.url, \"/api/jobs/MASTER.namea\")\n assert_equal(identifier.type, TronObjectType.job)\n\n def test_get_url_from_identifier_job(self):\n identifier = get_object_type_from_identifier(\n self.index,\n \"MASTER.namea\",\n )\n assert_equal(identifier.url, \"/api/jobs/MASTER.namea\")\n assert_equal(identifier.type, TronObjectType.job)\n\n def test_get_url_from_identifier_job_run(self):\n identifier = get_object_type_from_identifier(\n self.index,\n \"MASTER.nameb.7\",\n )\n assert_equal(identifier.url, \"/api/jobs/MASTER.nameb/7\")\n assert_equal(identifier.type, TronObjectType.job_run)\n\n def test_get_url_from_identifier_action_run(self):\n identifier = get_object_type_from_identifier(\n self.index,\n \"MASTER.nameb.7.run\",\n )\n assert_equal(identifier.url, \"/api/jobs/MASTER.nameb/7/run\")\n assert_equal(identifier.type, TronObjectType.action_run)\n\n def test_get_url_from_identifier_job_no_namespace_not_master(self):\n identifier = get_object_type_from_identifier(self.index, \"nameg\")\n assert_equal(identifier.url, \"/api/jobs/OTHER.nameg\")\n assert_equal(identifier.type, TronObjectType.job)\n\n def test_get_url_from_identifier_no_match(self):\n exc = assert_raises(\n ValueError,\n get_object_type_from_identifier,\n self.index,\n \"MASTER.namec\",\n )\n assert_in(\"namec\", str(exc))\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/commands/cmd_utils_test.py", "content": "import argparse\nfrom unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_in\nfrom testifycompat import setup_teardown\nfrom testifycompat import TestCase\nfrom tron.commands import cmd_utils\n\n\nclass TestGetConfig(TestCase):\n @setup_teardown\n def patch_environment(self):\n with mock.patch(\"tron.commands.cmd_utils.opener\", autospec=True) as self.mock_opener, mock.patch(\n \"tron.commands.cmd_utils.yaml\", autospec=True\n ) as self.mock_yaml:\n yield\n\n def test_read_config_missing(self):\n self.mock_opener.side_effect = IOError\n assert_equal(cmd_utils.read_config(), {})\n\n def test_read_config(self):\n assert_equal(cmd_utils.read_config(), self.mock_yaml.load.return_value)\n\n @mock.patch(\"tron.commands.cmd_utils.os.access\", autospec=True)\n def test_get_client_config(self, mock_access):\n mock_access.return_value = False\n config = cmd_utils.get_client_config()\n assert_equal(mock_access.call_count, 2)\n assert_equal(config, {})\n\n def test_filter_jobs_actions_runs_with_nothing(self):\n inputs = [\n \"M.foo\",\n \"M.foo.1\",\n \"M.foo.1.action1\",\n \"M.foo.2.action1\",\n \"M.bar\",\n \"M.bar.1.action\",\n ]\n prefix = \"\"\n expected = [\"M.foo\", \"M.bar\"]\n assert_equal(\n cmd_utils.filter_jobs_actions_runs(\n prefix,\n inputs,\n ),\n expected,\n )\n\n def test_filter_jobs_actions_runs_with_almost_a_job(self):\n inputs = [\n \"M.foo\",\n \"M.foo.1\",\n \"M.foo.1.action1\",\n \"M.foo.2.action1\",\n \"M.bar.1.action\",\n ]\n prefix = \"M.f\"\n expected = [\"M.foo\"]\n assert_equal(\n cmd_utils.filter_jobs_actions_runs(\n prefix,\n inputs,\n ),\n expected,\n )\n\n def test_filter_jobs_actions_runs_with_a_job_run(self):\n inputs = [\n \"M.foo\",\n \"M.foo.1\",\n \"M.foo.1.action1\",\n \"M.foo.2\",\n \"M.foo.2.action1\",\n \"M.bar.1.action\",\n ]\n prefix = \"M.foo.\"\n expected = [\"M.foo.1\", \"M.foo.2\"]\n assert_equal(\n cmd_utils.filter_jobs_actions_runs(\n prefix,\n inputs,\n ),\n expected,\n )\n\n def test_filter_jobs_actions_runs_with_a_job_run_and_id(self):\n inputs = [\n \"M.foo\",\n \"M.foo.1\",\n \"M.foo.1.action1\",\n \"M.foo.2.action1\",\n \"M.bar.1.action\",\n ]\n prefix = \"M.foo.1\"\n expected = [\"M.foo.1\", \"M.foo.1.action1\"]\n assert_equal(\n cmd_utils.filter_jobs_actions_runs(\n prefix,\n inputs,\n ),\n expected,\n )\n\n\nclass TestBuildOptionParser(TestCase):\n def test_build_option_parser(self):\n \"\"\"Assert that we don't set default options so that we can load\n the defaults from the config.\n \"\"\"\n usage = \"Something\"\n epilog = \"Something\"\n argparse.ArgumentParser = mock.Mock()\n parser = cmd_utils.build_option_parser(\n usage=usage,\n epilog=epilog,\n )\n argparse.ArgumentParser.assert_called_with(\n usage=usage,\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=epilog,\n )\n assert parser.add_argument.call_count == 5\n\n args = [call[1] for call in parser.add_argument.mock_calls]\n expected = [\n (\"--version\",),\n (\"-v\", \"--verbose\"),\n (\"--server\",),\n (\"--cluster_name\",),\n (\"-s\", \"--save\"),\n ]\n assert args == expected\n\n defaults = [call[2].get(\"default\") for call in parser.add_argument.mock_calls]\n assert defaults == [None, None, None, None, None]\n\n\nclass TestSuggestions(TestCase):\n def test_suggest_possibilities_none(self):\n expected = \"\"\n actual = cmd_utils.suggest_possibilities(word=\"FOO\", possibilities=[])\n assert_equal(actual, expected)\n\n def test_suggest_possibilities_many(self):\n expected = \"FOOO, FOOBAR\"\n actual = cmd_utils.suggest_possibilities(\n word=\"FOO\",\n possibilities=[\"FOOO\", \"FOOBAR\"],\n )\n assert_in(expected, actual)\n\n def test_suggest_possibilities_one(self):\n expected = \"FOOBAR?\"\n actual = cmd_utils.suggest_possibilities(\n word=\"FOO\",\n possibilities=[\"FOOBAR\", \"BAZ\"],\n )\n assert_in(expected, actual)\n" }, { "path": "tests/commands/display_test.py", "content": "from unittest import mock\n\nfrom testifycompat import setup\nfrom testifycompat import setup_teardown\nfrom testifycompat import TestCase\nfrom tron.commands import display\nfrom tron.commands.display import DisplayActionRuns\nfrom tron.commands.display import DisplayJobRuns\nfrom tron.commands.display import DisplayJobs\nfrom tron.core import actionrun\nfrom tron.core import job\n\n\nclass TestDisplayJobRuns(TestCase):\n @setup\n def setup_data(self):\n self.data = [\n dict(\n id=\"something.23\",\n state=\"FAIL\",\n node=mock.MagicMock(),\n run_num=23,\n run_time=\"2012-01-20 23:11:23\",\n start_time=\"2012-01-20 23:11:23\",\n end_time=\"2012-02-21 23:10:10\",\n duration=\"2 days\",\n manual=False,\n ),\n dict(\n id=\"something.55\",\n state=\"QUE\",\n node=mock.MagicMock(),\n run_num=55,\n run_time=\"2012-01-20 23:11:23\",\n start_time=\"2012-01-20 23:11:23\",\n end_time=\"\",\n duration=\"\",\n manual=False,\n ),\n ]\n\n def test_format(self):\n out = DisplayJobRuns().format(self.data)\n lines = out.split(\"\\n\")\n assert len(lines) == 7\n\n\nclass TestDisplayJobs(TestCase):\n @setup\n def setup_data(self):\n self.data = [\n dict(\n name=\"important_things\",\n status=\"running\",\n scheduler=mock.MagicMock(),\n last_success=None,\n ),\n dict(\n name=\"other_thing\",\n status=\"enabled\",\n scheduler=mock.MagicMock(),\n last_success=\"2012-01-23 10:23:23\",\n ),\n ]\n\n def do_format(self):\n out = DisplayJobs().format(self.data)\n lines = out.split(\"\\n\")\n return lines\n\n def test_format(self):\n lines = self.do_format()\n assert len(lines) == 5\n\n\nclass TestDisplayActions(TestCase):\n @setup\n def setup_data(self):\n self.data = {\n \"id\": \"something.23\",\n \"state\": \"UNKWN\",\n \"node\": {\n \"hostname\": \"something\",\n \"username\": \"a\",\n },\n \"run_time\": \"sometime\",\n \"start_time\": \"sometime\",\n \"end_time\": \"sometime\",\n \"manual\": False,\n \"runs\": [\n dict(\n id=\"something.23.run_other_thing\",\n state=\"unknown\",\n start_time=\"2012-01-23 10:10:10.123456\",\n end_time=\"\",\n duration=\"\",\n run_time=\"sometime\",\n ),\n dict(\n id=\"something.1.run_foo\",\n state=\"failed\",\n start_time=\"2012-01-23 10:10:10.123456\",\n end_time=\"2012-01-23 10:40:10.123456\",\n duration=\"1234.123456\",\n run_time=\"sometime\",\n ),\n dict(\n id=\"something.23.run_other_thing\",\n state=\"queued\",\n start_time=\"2012-01-23 10:10:10.123456\",\n end_time=\"\",\n duration=\"\",\n run_time=\"sometime\",\n ),\n dict(\n id=\"something.42.weird_run\",\n state=\"unknown\",\n start_time=None,\n end_time=None,\n duration=\"\",\n run_time=None,\n ),\n dict(\n id=\"something.43.skipped\",\n state=\"skipped\",\n start_time=\"2019-07-15 18:12:05\",\n end_time=\"2019-07-16 01:31:50\",\n duration=\"7:19:44.506211\",\n ),\n dict(\n id=\"something.43.running\",\n state=\"running\",\n start_time=\"2019-07-15 18:12:05\",\n end_time=None,\n duration=\"7:19:44.506211\",\n ),\n ],\n }\n self.details = {\n \"id\": \"something.1.foo\",\n \"state\": \"FAIL\",\n \"node\": \"localhost\",\n \"stdout\": [\"Blah\", \"blah\", \"blah\"],\n \"stderr\": [\"Crash\", \"and\", \"burn\"],\n \"command\": \"/bin/bash ./runme.sh now\",\n \"raw_command\": \"bash runme.sh now\",\n \"requirements\": [\".run_first_job\"],\n }\n\n def format_lines(self):\n out = DisplayActionRuns().format(self.data)\n return out.split(\"\\n\")\n\n def test_format(self):\n lines = self.format_lines()\n assert len(lines) == 16, \"\\n\".join(lines)\n\n\nclass TestAddColorForState(TestCase):\n @setup_teardown\n def enable_color(self):\n with display.Color.enable():\n yield\n\n def test_add_red(self):\n text = display.add_color_for_state(actionrun.ActionRun.FAILED)\n assert text.startswith(display.Color.colors[\"red\"]), text\n\n def test_add_green(self):\n text = display.add_color_for_state(actionrun.ActionRun.RUNNING)\n assert text.startswith(display.Color.colors[\"green\"]), text\n\n def test_add_blue(self):\n text = display.add_color_for_state(job.Job.STATUS_DISABLED)\n assert text.startswith(display.Color.colors[\"blue\"]), text\n\n\nclass TestDisplayNode(TestCase):\n\n node_source = {\n \"name\": \"name\",\n \"hostname\": \"hostname\",\n \"username\": \"username\",\n }\n\n def test_display_node(self):\n result = display.display_node(self.node_source)\n assert result == \"username@hostname\"\n\n def test_display_node_pool(self):\n source = {\"name\": \"name\", \"nodes\": [self.node_source]}\n result = display.display_node_pool(source)\n assert result == \"name (1 node(s))\"\n" }, { "path": "tests/commands/retry_test.py", "content": "import random\nfrom unittest import mock\n\nimport pytest\n\nfrom tron.commands import client\nfrom tron.commands import retry\n\n\nasync def _empty_coro(*args, **kwargs):\n return None\n\n\n@pytest.fixture(autouse=True)\ndef mock_sleep():\n with mock.patch(\"asyncio.sleep\", _empty_coro, autospec=None):\n yield\n\n\n@pytest.fixture(autouse=True)\ndef mock_client():\n with mock.patch.object(client, \"Client\", autospec=True) as m:\n m.return_value.url_base = \"http://localhost\"\n yield m\n\n\n@pytest.fixture(autouse=True)\ndef mock_urlopen(): # prevent any requests from being made\n with mock.patch(\"urllib.request.urlopen\", autospec=True) as m:\n yield m\n\n\n@pytest.fixture\ndef mock_client_request():\n with mock.patch.object(client, \"request\", autospec=True) as m:\n m.return_value = mock.Mock(error=False, content={}) # response\n yield m\n\n\n@mock.patch.object(\n client,\n \"get_object_type_from_identifier\",\n return_value=client.TronObjectIdentifier(\"JOB_RUN\", \"/a_job_run\"),\n autospec=True,\n)\ndef test_retry_action_init_not_an_action(mock_get_obj_type, mock_client):\n tron_client = mock_client.return_value\n with pytest.raises(ValueError):\n retry.RetryAction(tron_client, \"a_fake_action_run\")\n\n\n@pytest.fixture\ndef fake_retry_action(mock_client):\n tron_client = mock_client.return_value\n tron_client.action_runs.return_value = dict(\n action_name=\"a_fake_action\",\n requirements=[\"required_action_0\", \"required_action_1\"],\n triggered_by=\"a_fake_trigger_0 (done), a_fake_trigger_1\",\n )\n tron_client.job_runs.return_value = dict(\n job_name=\"a_fake_job\",\n run_num=1234,\n runs=[\n dict(action_name=\"required_action_0\", state=\"succeeded\"),\n dict(action_name=\"non_required_action\", state=\"succeeded\"),\n dict(action_name=\"required_action_1\", state=\"failed\"),\n dict(action_name=\"upstream_action_0\", trigger_downstreams=\"a_fake_trigger_0\"),\n dict(action_name=\"upstream_action_1\", trigger_downstreams=\"a_fake_trigger_1\"),\n tron_client.action_runs.return_value,\n ],\n )\n\n with mock.patch.object(\n client,\n \"get_object_type_from_identifier\",\n side_effect=[\n client.TronObjectIdentifier(\"ACTION_RUN\", \"/a_fake_job/0/a_fake_action\"),\n client.TronObjectIdentifier(\"JOB_RUN\", \"/a_fake_job/0\"),\n ],\n autospec=True,\n ):\n yield retry.RetryAction(tron_client, \"a_fake_job.0.a_fake_action\", use_latest_command=True)\n\n\ndef test_retry_action_init_ok(fake_retry_action):\n assert fake_retry_action.retry_params == dict(command=\"retry\", use_latest_command=1)\n assert fake_retry_action.full_action_name == \"a_fake_job.0.a_fake_action\"\n fake_retry_action.tron_client.action_runs.assert_called_once_with(\n \"/a_fake_job/0/a_fake_action\",\n num_lines=0,\n )\n assert fake_retry_action.action_name == \"a_fake_action\"\n assert fake_retry_action.action_run_id.url == \"/a_fake_job/0/a_fake_action\"\n fake_retry_action.tron_client.job_runs.assert_called_once_with(\"/a_fake_job/0\")\n assert fake_retry_action.job_run_name == \"a_fake_job.0\"\n assert fake_retry_action.job_run_id.url == \"/a_fake_job/0\"\n assert fake_retry_action._required_action_indices == {\"required_action_0\": 0, \"required_action_1\": 2}\n\n\ndef test_check_trigger_statuses(fake_retry_action, event_loop):\n expected = dict(a_fake_trigger_0=True, a_fake_trigger_1=False)\n assert expected == event_loop.run_until_complete(fake_retry_action.check_trigger_statuses())\n assert fake_retry_action.tron_client.action_runs.call_args_list[1] == mock.call( # 0th call is in init\n \"/a_fake_job/0/a_fake_action\",\n num_lines=0,\n )\n\n\ndef test_check_required_actions_statuses(fake_retry_action, event_loop):\n expected = dict(required_action_0=True, required_action_1=False)\n assert expected == event_loop.run_until_complete(fake_retry_action.check_required_actions_statuses())\n assert fake_retry_action.tron_client.job_runs.call_args_list[1] == mock.call(\"/a_fake_job/0\") # 0th call is in init\n\n\n@pytest.mark.parametrize(\n \"expected,triggered_by,required_action_1_state\",\n [\n (False, \"a_fake_trigger_0 (done), a_fake_trigger_1\", \"skipped\"), # unpublished triggers\n (False, \"a_fake_trigger_0 (done), a_fake_trigger_1 (done)\", \"failed\"), # required not succeeded\n (True, \"a_fake_trigger_0 (done), a_fake_trigger_1 (done)\", \"succeeded\"), # all done\n ],\n)\ndef test_can_retry(fake_retry_action, event_loop, expected, triggered_by, required_action_1_state):\n fake_retry_action.tron_client.action_runs.return_value[\"triggered_by\"] = triggered_by\n fake_retry_action.tron_client.job_runs.return_value[\"runs\"][2][\"state\"] = required_action_1_state\n assert expected == event_loop.run_until_complete(fake_retry_action.can_retry())\n\n\ndef test_wait_for_deps_timeout(fake_retry_action, event_loop):\n assert not event_loop.run_until_complete(fake_retry_action.wait_for_deps(deps_timeout_s=3, poll_interval_s=1))\n assert fake_retry_action._elapsed.seconds == 3\n assert fake_retry_action.tron_client.action_runs.call_count == 5 # 1 in init, 4 in this test\n\n\ndef test_wait_for_deps_all_deps_done(fake_retry_action, event_loop):\n fake_retry_action.tron_client.job_runs.return_value[\"runs\"][2][\"state\"] = \"skipped\"\n fake_retry_action.tron_client.action_runs.return_value = None\n triggered_by_results = [\n \"a_fake_trigger_0 (done), a_fake_trigger_1\",\n \"a_fake_trigger_0 (done), a_fake_trigger_1\",\n \"a_fake_trigger_0 (done), a_fake_trigger_1 (done)\",\n ]\n fake_retry_action.tron_client.action_runs.side_effect = [\n dict(\n action_name=\"a_fake_action\",\n requirements=[\"required_action_0\", \"required_action_1\"],\n triggered_by=r,\n )\n for r in triggered_by_results\n ]\n\n assert event_loop.run_until_complete(fake_retry_action.wait_for_deps(deps_timeout_s=3, poll_interval_s=1))\n # 3rd triggered_by result returned on check at 2nd second\n assert fake_retry_action._elapsed.seconds == 2\n assert fake_retry_action.tron_client.action_runs.call_count == 4 # 1 in init, 3 in this test\n\n\n@pytest.mark.parametrize(\"expected,error\", [(False, True), (True, False)])\ndef test_issue_retry(fake_retry_action, mock_client_request, event_loop, expected, error):\n mock_client_request.return_value.error = error\n assert expected == event_loop.run_until_complete(fake_retry_action.issue_retry())\n assert expected == fake_retry_action.succeeded\n\n\ndef test_wait_for_retry_deps_not_done(fake_retry_action, mock_client_request, event_loop):\n assert not event_loop.run_until_complete(\n fake_retry_action.wait_and_retry(deps_timeout_s=10, poll_interval_s=1, jitter=True),\n )\n assert fake_retry_action._elapsed.seconds == 10 # timeout\n mock_client_request.assert_not_called() # retry not attempted\n\n\ndef test_wait_for_retry_deps_done(fake_retry_action, mock_client_request, event_loop):\n fake_retry_action.tron_client.job_runs.return_value[\"runs\"][2][\"state\"] = \"skipped\"\n fake_retry_action.tron_client.action_runs.return_value[\n \"triggered_by\"\n ] = \"a_fake_trigger_0 (done), a_fake_trigger_1 (done)\"\n mock_client_request.return_value.error = False\n random.seed(1) # init delay is 1s\n\n assert event_loop.run_until_complete(\n fake_retry_action.wait_and_retry(deps_timeout_s=10, poll_interval_s=5, jitter=True),\n )\n assert fake_retry_action._elapsed.seconds == 1 # init delay only\n mock_client_request.assert_called_once_with(\n \"http://localhost/a_fake_job/0/a_fake_action\",\n data=dict(command=\"retry\", use_latest_command=1),\n user_attribution=True,\n )\n\n\n@mock.patch.object(retry, \"RetryAction\", autospec=True)\ndef test_retry_actions(mock_retry_action, mock_client, event_loop):\n mock_wait_and_retry = mock_retry_action.return_value.wait_and_retry\n mock_wait_and_retry.return_value = _empty_coro()\n\n r_actions = retry.retry_actions(\n \"http://localhost\",\n [\"a_job.0.an_action_0\", \"another_job.1.an_action_1\"],\n use_latest_command=True,\n deps_timeout_s=4,\n )\n\n assert r_actions == [mock_retry_action.return_value] * 2\n assert mock_retry_action.call_args_list == [\n mock.call(mock_client.return_value, \"a_job.0.an_action_0\", use_latest_command=True),\n mock.call(mock_client.return_value, \"another_job.1.an_action_1\", use_latest_command=True),\n ]\n assert mock_wait_and_retry.call_args_list == [\n mock.call(deps_timeout_s=4, jitter=False),\n mock.call(deps_timeout_s=4),\n ]\n" }, { "path": "tests/config/__init__.py", "content": "" }, { "path": "tests/config/config_parse_test.py", "content": "import datetime\nimport os\nimport shutil\nimport tempfile\nfrom unittest import mock\n\nimport pytest\nimport pytz\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_in\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import teardown\nfrom testifycompat import TestCase\nfrom tests.assertions import assert_raises\nfrom tron.config import config_parse\nfrom tron.config import config_utils\nfrom tron.config import ConfigError\nfrom tron.config import schedule_parse\nfrom tron.config import schema\nfrom tron.config.config_parse import build_format_string_validator\nfrom tron.config.config_parse import CLEANUP_ACTION_NAME\nfrom tron.config.config_parse import valid_cleanup_action_name\nfrom tron.config.config_parse import valid_config\nfrom tron.config.config_parse import valid_job\nfrom tron.config.config_parse import valid_node_pool\nfrom tron.config.config_parse import valid_output_stream_dir\nfrom tron.config.config_parse import validate_fragment\nfrom tron.config.config_utils import NullConfigContext\nfrom tron.config.schedule_parse import ConfigDailyScheduler\nfrom tron.config.schema import ConfigNodeAffinity\nfrom tron.config.schema import MASTER_NAMESPACE\n\nBASE_CONFIG = dict(\n ssh_options=dict(agent=False, identities=[\"tests/test_id_rsa\"]),\n time_zone=\"EST\",\n output_stream_dir=\"/tmp\",\n nodes=[\n dict(name=\"node0\", hostname=\"node0\"),\n dict(name=\"node1\", hostname=\"node1\"),\n ],\n node_pools=[dict(name=\"NodePool\", nodes=[\"node0\", \"node1\"])],\n)\n\n\ndef make_ssh_options():\n return schema.ConfigSSHOptions(\n agent=False,\n identities=(\"tests/test_id_rsa\",),\n known_hosts_file=None,\n connect_timeout=30,\n idle_connection_timeout=3600,\n jitter_min_load=4,\n jitter_max_delay=20,\n jitter_load_factor=1,\n )\n\n\ndef make_mock_schedule():\n return ConfigDailyScheduler(\n days=set(),\n hour=0,\n minute=0,\n second=0,\n original=\"00:00:00 \",\n jitter=None,\n )\n\n\ndef make_command_context():\n return {\n \"python\": \"/usr/bin/python\",\n \"batch_dir\": \"/tron/batch/test/foo\",\n }\n\n\ndef make_nodes():\n return {\n \"node0\": schema.ConfigNode(\n name=\"node0\",\n username=\"foo\",\n hostname=\"node0\",\n port=22,\n ),\n \"node1\": schema.ConfigNode(\n name=\"node1\",\n username=\"foo\",\n hostname=\"node1\",\n port=22,\n ),\n }\n\n\ndef make_node_pools():\n return {\n \"NodePool\": schema.ConfigNodePool(\n nodes=(\"node0\", \"node1\"),\n name=\"NodePool\",\n ),\n }\n\n\ndef make_mesos_options():\n return schema.ConfigMesos(\n master_address=None,\n master_port=5050,\n secret_file=None,\n role=\"*\",\n principal=\"tron\",\n enabled=False,\n default_volumes=(),\n dockercfg_location=None,\n offer_timeout=300,\n )\n\n\ndef make_k8s_options():\n return schema.ConfigKubernetes(enabled=False, non_retryable_exit_codes=(), default_volumes=())\n\n\ndef make_action(**kwargs):\n kwargs.setdefault(\"name\", \"action\"),\n kwargs.setdefault(\"command\", \"command\")\n kwargs.setdefault(\"executor\", schema.ExecutorTypes.ssh.value)\n kwargs.setdefault(\"requires\", ())\n kwargs.setdefault(\"expected_runtime\", datetime.timedelta(1))\n return schema.ConfigAction(**kwargs)\n\n\ndef make_cleanup_action(**kwargs):\n kwargs.setdefault(\"name\", \"cleanup\"),\n kwargs.setdefault(\"command\", \"command\")\n kwargs.setdefault(\"executor\", schema.ExecutorTypes.ssh.value)\n kwargs.setdefault(\"expected_runtime\", datetime.timedelta(1))\n return schema.ConfigCleanupAction(**kwargs)\n\n\ndef make_job(**kwargs):\n kwargs.setdefault(\"namespace\", \"MASTER\")\n kwargs.setdefault(\"name\", f\"{kwargs['namespace']}.job_name\")\n kwargs.setdefault(\"node\", \"node0\")\n kwargs.setdefault(\"enabled\", True)\n kwargs.setdefault(\"monitoring\", {})\n kwargs.setdefault(\n \"schedule\",\n schedule_parse.ConfigDailyScheduler(\n days=set(),\n hour=16,\n minute=30,\n second=0,\n original=\"16:30:00 \",\n jitter=None,\n ),\n )\n kwargs.setdefault(\"actions\", {\"action\": make_action()})\n kwargs.setdefault(\"queueing\", True)\n kwargs.setdefault(\"run_limit\", 50)\n kwargs.setdefault(\"all_nodes\", False)\n kwargs.setdefault(\"cleanup_action\", make_cleanup_action())\n kwargs.setdefault(\"max_runtime\")\n kwargs.setdefault(\"allow_overlap\", False)\n kwargs.setdefault(\"time_zone\", None)\n kwargs.setdefault(\"expected_runtime\", datetime.timedelta(0, 3600))\n kwargs.setdefault(\"use_k8s\", False)\n return schema.ConfigJob(**kwargs)\n\n\ndef make_master_jobs():\n return {\n \"MASTER.test_job0\": make_job(\n name=\"MASTER.test_job0\",\n schedule=make_mock_schedule(),\n expected_runtime=datetime.timedelta(1),\n ),\n \"MASTER.test_job1\": make_job(\n name=\"MASTER.test_job1\",\n schedule=schedule_parse.ConfigDailyScheduler(\n days={1, 3, 5},\n hour=0,\n minute=30,\n second=0,\n original=\"00:30:00 MWF\",\n jitter=None,\n ),\n actions={\n \"action\": make_action(\n requires=(\"action1\",),\n expected_runtime=datetime.timedelta(0, 7200),\n ),\n \"action1\": make_action(\n name=\"action1\",\n expected_runtime=datetime.timedelta(0, 7200),\n ),\n },\n time_zone=pytz.timezone(\"Pacific/Auckland\"),\n expected_runtime=datetime.timedelta(1),\n cleanup_action=None,\n allow_overlap=True,\n ),\n \"MASTER.test_job2\": make_job(\n name=\"MASTER.test_job2\",\n node=\"node1\",\n actions={\n \"action2_0\": make_action(\n name=\"action2_0\",\n command=\"test_command2.0\",\n ),\n },\n time_zone=pytz.timezone(\"Pacific/Auckland\"),\n expected_runtime=datetime.timedelta(1),\n cleanup_action=None,\n ),\n \"MASTER.test_job_actions_dict\": make_job(\n name=\"MASTER.test_job_actions_dict\",\n node=\"node1\",\n schedule=make_mock_schedule(),\n actions={\n \"action\": make_action(),\n \"action1\": make_action(name=\"action1\"),\n \"action2\": make_action(\n name=\"action2\",\n requires=(\"action\", \"action1\"),\n node=\"node0\",\n ),\n },\n cleanup_action=None,\n expected_runtime=datetime.timedelta(1),\n ),\n \"MASTER.test_job4\": make_job(\n name=\"MASTER.test_job4\",\n node=\"NodePool\",\n schedule=schedule_parse.ConfigDailyScheduler(\n original=\"00:00:00 \",\n hour=0,\n minute=0,\n second=0,\n days=set(),\n jitter=None,\n ),\n all_nodes=True,\n enabled=False,\n cleanup_action=None,\n expected_runtime=datetime.timedelta(1),\n ),\n \"MASTER.test_job_mesos\": make_job(\n name=\"MASTER.test_job_mesos\",\n node=\"NodePool\",\n schedule=schedule_parse.ConfigDailyScheduler(\n original=\"00:00:00 \",\n hour=0,\n minute=0,\n second=0,\n days=set(),\n jitter=None,\n ),\n actions={\n \"action_mesos\": make_action(\n name=\"action_mesos\",\n command=\"test_command_mesos\",\n executor=schema.ExecutorTypes.mesos.value,\n cpus=0.1,\n mem=100,\n disk=600,\n docker_image=\"container:latest\",\n ),\n },\n cleanup_action=None,\n expected_runtime=datetime.timedelta(1),\n ),\n \"MASTER.test_job_k8s\": make_job(\n name=\"MASTER.test_job_k8s\",\n node=\"NodePool\",\n schedule=schedule_parse.ConfigDailyScheduler(\n original=\"00:00:00 \",\n hour=0,\n minute=0,\n second=0,\n days=set(),\n jitter=None,\n ),\n actions={\n \"action_k8s\": make_action(\n name=\"action_k8s\",\n command=\"test_command_k8s\",\n executor=schema.ExecutorTypes.kubernetes.value,\n cpus=0.1,\n mem=100,\n disk=600,\n cap_add=[\"KILL\"],\n cap_drop=[\"CHOWN\", \"KILL\"],\n docker_image=\"container:latest\",\n secret_env=dict(\n TEST_SECRET=schema.ConfigSecretSource(secret_name=\"tron-secret-test-secret--1\", key=\"secret_1\")\n ),\n secret_volumes=(\n schema.ConfigSecretVolume(\n secret_volume_name=\"abc\",\n secret_name=\"secret1\",\n container_path=\"/b/c\",\n default_mode=\"0644\",\n items=(schema.ConfigSecretVolumeItem(key=\"secret1\", path=\"abcd\", mode=\"777\"),),\n ),\n ),\n projected_sa_volumes=(\n schema.ConfigProjectedSAVolume(\n container_path=\"/var/secrets/whatever\",\n audience=\"foo.bar\",\n expiration_seconds=1800,\n ),\n ),\n node_selectors={\"yelp.com/pool\": \"default\"},\n node_affinities=(\n ConfigNodeAffinity(\n key=\"instance_type\",\n operator=\"In\",\n value=(\"a1.1xlarge\",),\n ),\n ),\n ),\n },\n cleanup_action=None,\n expected_runtime=datetime.timedelta(1),\n ),\n }\n\n\ndef make_tron_config(\n action_runner=None,\n output_stream_dir=\"/tmp\",\n command_context=None,\n ssh_options=None,\n time_zone=pytz.timezone(\"EST\"),\n state_persistence=config_parse.DEFAULT_STATE_PERSISTENCE,\n nodes=None,\n node_pools=None,\n jobs=None,\n mesos_options=None,\n k8s_options=None,\n read_json=False,\n):\n return schema.TronConfig(\n action_runner=action_runner or {},\n output_stream_dir=output_stream_dir,\n command_context=command_context or dict(batch_dir=\"/tron/batch/test/foo\", python=\"/usr/bin/python\"),\n ssh_options=ssh_options or make_ssh_options(),\n time_zone=time_zone,\n state_persistence=state_persistence,\n nodes=nodes or make_nodes(),\n node_pools=node_pools or make_node_pools(),\n jobs=jobs or make_master_jobs(),\n mesos_options=mesos_options or make_mesos_options(),\n k8s_options=k8s_options or make_k8s_options(),\n read_json=read_json,\n )\n\n\ndef make_named_tron_config(jobs=None):\n return schema.NamedTronConfig(jobs=jobs or make_master_jobs())\n\n\nclass ConfigTestCase(TestCase):\n JOBS_CONFIG = dict(\n jobs=[\n dict(\n name=\"test_job0\",\n node=\"node0\",\n schedule=\"daily 00:00:00\",\n actions=[dict(name=\"action\", command=\"command\")],\n cleanup_action=dict(command=\"command\"),\n ),\n dict(\n name=\"test_job1\",\n node=\"node0\",\n schedule=\"daily 00:30:00 MWF\",\n allow_overlap=True,\n time_zone=\"Pacific/Auckland\",\n actions=[\n dict(\n name=\"action\",\n command=\"command\",\n requires=[\"action1\"],\n expected_runtime=\"2h\",\n ),\n dict(\n name=\"action1\",\n command=\"command\",\n expected_runtime=\"2h\",\n ),\n ],\n ),\n dict(\n name=\"test_job2\",\n node=\"node1\",\n schedule=\"daily 16:30:00\",\n expected_runtime=\"1d\",\n time_zone=\"Pacific/Auckland\",\n actions=[dict(name=\"action2_0\", command=\"test_command2.0\")],\n ),\n dict(\n name=\"test_job_actions_dict\",\n node=\"node1\",\n schedule=\"daily 00:00:00 \",\n actions=dict(\n action=dict(command=\"command\"),\n action1=dict(command=\"command\"),\n action2=dict(\n node=\"node0\",\n command=\"command\",\n requires=[\"action\", \"action1\"],\n ),\n ),\n ),\n dict(\n name=\"test_job4\",\n node=\"NodePool\",\n all_nodes=True,\n schedule=\"daily\",\n enabled=False,\n actions=[dict(name=\"action\", command=\"command\")],\n ),\n dict(\n name=\"test_job_mesos\",\n node=\"NodePool\",\n schedule=\"daily\",\n actions=[\n dict(\n name=\"action_mesos\",\n executor=\"mesos\",\n command=\"test_command_mesos\",\n cpus=0.1,\n mem=100,\n disk=600,\n docker_image=\"container:latest\",\n ),\n ],\n ),\n dict(\n name=\"test_job_k8s\",\n node=\"NodePool\",\n schedule=\"daily\",\n actions=[\n dict(\n name=\"action_k8s\",\n executor=\"kubernetes\",\n command=\"test_command_k8s\",\n cpus=0.1,\n mem=100,\n disk=600,\n docker_image=\"container:latest\",\n secret_env=dict(TEST_SECRET=dict(secret_name=\"tron-secret-test-secret--1\", key=\"secret_1\")),\n secret_volumes=[\n dict(\n secret_volume_name=\"abc\",\n secret_name=\"secret1\",\n container_path=\"/b/c\",\n default_mode=\"0644\",\n items=[\n dict(key=\"secret1\", path=\"abcd\", mode=\"777\"),\n ],\n ),\n ],\n projected_sa_volumes=[\n dict(\n container_path=\"/var/secrets/whatever\",\n audience=\"foo.bar\",\n expiration_seconds=1800,\n ),\n ],\n cap_add=[\"KILL\"],\n cap_drop=[\"CHOWN\", \"KILL\"],\n node_selectors={\"yelp.com/pool\": \"default\"},\n node_affinities=[{\"key\": \"instance_type\", \"operator\": \"In\", \"value\": [\"a1.1xlarge\"]}],\n ),\n ],\n ),\n ],\n )\n\n config = dict(\n command_context=dict(\n batch_dir=\"/tron/batch/test/foo\",\n python=\"/usr/bin/python\",\n ),\n **BASE_CONFIG,\n **JOBS_CONFIG,\n )\n\n @mock.patch.dict(\"tron.config.config_parse.ValidateNode.defaults\")\n def test_attributes(self):\n config_parse.ValidateNode.defaults[\"username\"] = \"foo\"\n expected = make_tron_config()\n\n test_config = valid_config(self.config)\n\n assert test_config.command_context == expected.command_context\n assert test_config.ssh_options == expected.ssh_options\n assert test_config.mesos_options == expected.mesos_options\n assert test_config.time_zone == expected.time_zone\n assert test_config.nodes == expected.nodes\n assert test_config.node_pools == expected.node_pools\n assert test_config.k8s_options == expected.k8s_options\n assert test_config.read_json == expected.read_json\n for key in [\"0\", \"1\", \"2\", \"_actions_dict\", \"4\", \"_mesos\"]:\n job_name = f\"MASTER.test_job{key}\"\n assert job_name in test_config.jobs, f\"{job_name} in test_config.jobs\"\n assert job_name in expected.jobs, f\"{job_name} in test_config.jobs\"\n assert test_config.jobs[job_name] == expected.jobs[job_name]\n assert test_config == expected\n\n def test_empty_node_test(self):\n valid_config(dict(nodes=None))\n\n\nclass TestNamedConfig(TestCase):\n config = ConfigTestCase.JOBS_CONFIG\n\n def test_attributes(self):\n expected = make_named_tron_config(\n jobs={\n \"test_job\": make_job(\n name=\"test_job\",\n namespace=\"test_namespace\",\n schedule=make_mock_schedule(),\n expected_runtime=datetime.timedelta(1),\n ),\n },\n )\n test_config = validate_fragment(\n \"test_namespace\",\n dict(\n jobs=[\n dict(\n name=\"test_job\",\n namespace=\"test_namespace\",\n node=\"node0\",\n schedule=\"daily 00:00:00 \",\n actions=[dict(name=\"action\", command=\"command\")],\n cleanup_action=dict(command=\"command\"),\n ),\n ],\n ),\n )\n assert_equal(test_config, expected)\n\n def test_attributes_with_master_context(self):\n expected = make_named_tron_config(\n jobs={\n \"test_namespace.test_job\": make_job(\n name=\"test_namespace.test_job\",\n namespace=\"test_namespace\",\n schedule=make_mock_schedule(),\n expected_runtime=datetime.timedelta(1),\n ),\n },\n )\n master_config = dict(\n nodes=[\n dict(\n name=\"node0\",\n hostname=\"node0\",\n ),\n ],\n node_pools=[\n dict(\n name=\"nodepool0\",\n nodes=[\"node0\"],\n ),\n ],\n )\n test_config = validate_fragment(\n \"test_namespace\",\n dict(\n jobs=[\n dict(\n name=\"test_job\",\n namespace=\"test_namespace\",\n node=\"node0\",\n schedule=\"daily 00:00:00\",\n actions=[dict(name=\"action\", command=\"command\")],\n cleanup_action=dict(command=\"command\"),\n ),\n ],\n ),\n master_config=master_config,\n )\n assert_equal(test_config, expected)\n\n def test_invalid_job_node_with_master_context(self):\n master_config = dict(\n nodes=[\n dict(\n name=\"node0\",\n hostname=\"node0\",\n ),\n ],\n )\n test_config = dict(\n jobs=[\n dict(\n name=\"test_job\",\n namespace=\"test_namespace\",\n node=\"node1\",\n schedule=\"daily 00:30:00 \",\n actions=[dict(name=\"action\", command=\"command\")],\n cleanup_action=dict(command=\"command\"),\n ),\n ],\n )\n expected_message = \"Unknown node name node1 at test_namespace.NamedConfigFragment.jobs.Job.test_job.node\"\n exception = assert_raises(\n ConfigError,\n validate_fragment,\n \"test_namespace\",\n test_config,\n master_config,\n )\n assert_in(expected_message, str(exception))\n\n def test_invalid_action_node_with_master_context(self):\n master_config = dict(\n nodes=[\n dict(\n name=\"node0\",\n hostname=\"node0\",\n ),\n ],\n node_pools=[\n dict(\n name=\"nodepool0\",\n nodes=[\"node0\"],\n ),\n ],\n )\n test_config = dict(\n jobs=[\n dict(\n name=\"test_job\",\n namespace=\"test_namespace\",\n node=\"node0\",\n schedule=\"daily 00:30:00 \",\n actions=[dict(name=\"action\", node=\"nodepool1\", command=\"command\")],\n cleanup_action=dict(command=\"command\"),\n ),\n ],\n )\n expected_message = \"Unknown node name nodepool1 at test_namespace.NamedConfigFragment.jobs.Job.test_job.actions.Action.action.node\"\n\n exception = assert_raises(\n ConfigError,\n validate_fragment,\n \"test_namespace\",\n test_config,\n master_config,\n )\n assert_in(expected_message, str(exception))\n\n\nclass TestJobConfig(TestCase):\n def test_no_actions(self):\n test_config = dict(\n jobs=[\n dict(name=\"test_job0\", node=\"node0\", schedule=\"daily 00:30:00 \"),\n ],\n **BASE_CONFIG,\n )\n\n expected_message = \"Job test_job0 is missing options: actions\"\n exception = assert_raises(\n ConfigError,\n valid_config,\n test_config,\n )\n assert_in(expected_message, str(exception))\n\n def test_empty_actions(self):\n test_config = dict(\n jobs=[\n dict(\n name=\"test_job0\",\n node=\"node0\",\n schedule=\"daily 00:30:00 \",\n actions=None,\n ),\n ],\n **BASE_CONFIG,\n )\n\n expected_message = \"Value at config.jobs.Job.test_job0.actions\"\n exception = assert_raises(\n ConfigError,\n valid_config,\n test_config,\n )\n assert_in(expected_message, str(exception))\n\n def test_dupe_names(self):\n test_config = dict(\n jobs=[\n dict(\n name=\"test_job0\",\n node=\"node0\",\n schedule=\"daily 00:30:00\",\n actions=[\n dict(name=\"action\", command=\"cmd\"),\n dict(name=\"action\", command=\"cmd\"),\n ],\n ),\n ],\n **BASE_CONFIG,\n )\n\n expected = \"Duplicate name action at config.jobs.Job.test_job0.actions\"\n exception = assert_raises(\n ConfigError,\n valid_config,\n test_config,\n )\n assert_in(expected, str(exception))\n\n def test_bad_requires(self):\n test_config = dict(\n jobs=[\n dict(\n name=\"test_job0\",\n node=\"node0\",\n schedule=\"daily 00:30:00\",\n actions=[dict(name=\"action\", command=\"cmd\")],\n ),\n dict(\n name=\"test_job1\",\n node=\"node0\",\n schedule=\"daily 00:30:00\",\n actions=[\n dict(\n name=\"action1\",\n command=\"cmd\",\n requires=[\"action\"],\n ),\n ],\n ),\n ],\n **BASE_CONFIG,\n )\n\n expected_message = \"jobs.MASTER.test_job1.action1 has a dependency \" '\"action\" that is not in the same job!'\n exception = assert_raises(\n ConfigError,\n valid_config,\n test_config,\n )\n assert_in(expected_message, str(exception))\n\n def test_circular_dependency(self):\n test_config = dict(\n jobs=[\n dict(\n name=\"test_job0\",\n node=\"node0\",\n schedule=\"daily 00:30:00\",\n actions=[\n dict(\n name=\"action1\",\n command=\"cmd\",\n requires=[\"action2\"],\n ),\n dict(\n name=\"action2\",\n command=\"cmd\",\n requires=[\"action1\"],\n ),\n ],\n ),\n ],\n **BASE_CONFIG,\n )\n\n expect = \"Circular dependency in job.MASTER.test_job0: action1 -> action2\"\n exception = assert_raises(\n ConfigError,\n valid_config,\n test_config,\n )\n assert_in(expect, str(exception))\n\n def test_circular_dependency_multiaction(self):\n test_config = dict(\n jobs=[\n dict(\n name=\"test_job0\",\n node=\"node0\",\n schedule=\"daily 00:30:00\",\n actions=[\n dict(\n name=\"action1\",\n command=\"cmd\",\n requires=[\"action2\"],\n ),\n dict(\n name=\"action2\",\n command=\"cmd\",\n requires=[\"action3\"],\n ),\n dict(\n name=\"action3\",\n command=\"cmd\",\n requires=[\"action4\"],\n ),\n dict(\n name=\"action4\",\n command=\"cmd\",\n requires=[\"action5\"],\n ),\n dict(\n name=\"action5\",\n command=\"cmd\",\n requires=[\"action3\"],\n ),\n ],\n ),\n ],\n **BASE_CONFIG,\n )\n\n expect = \"Circular dependency in job.MASTER.test_job0: action3 -> action4 -> action5\"\n exception = assert_raises(\n ConfigError,\n valid_config,\n test_config,\n )\n assert_in(expect, str(exception))\n\n def test_config_cleanup_name_collision(self):\n test_config = dict(\n jobs=[\n dict(\n name=\"test_job0\",\n node=\"node0\",\n schedule=\"daily 00:30:00\",\n actions=[\n dict(name=CLEANUP_ACTION_NAME, command=\"cmd\"),\n ],\n ),\n ],\n **BASE_CONFIG,\n )\n expected_message = \"config.jobs.Job.test_job0.actions.Action.cleanup.name\"\n exception = assert_raises(\n ConfigError,\n valid_config,\n test_config,\n )\n assert_in(expected_message, str(exception))\n\n def test_config_cleanup_action_name(self):\n test_config = dict(\n jobs=[\n dict(\n name=\"test_job0\",\n node=\"node0\",\n schedule=\"daily 00:30:00\",\n actions=[\n dict(name=\"action\", command=\"cmd\"),\n ],\n cleanup_action=dict(name=\"gerald\", command=\"cmd\"),\n ),\n ],\n **BASE_CONFIG,\n )\n\n expected_msg = \"Cleanup actions cannot have custom names\"\n exception = assert_raises(\n ConfigError,\n valid_config,\n test_config,\n )\n assert_in(expected_msg, str(exception))\n\n def test_config_cleanup_requires(self):\n test_config = dict(\n jobs=[\n dict(\n name=\"test_job0\",\n node=\"node0\",\n schedule=\"daily 00:30:00\",\n actions=[\n dict(name=\"action\", command=\"cmd\"),\n ],\n cleanup_action=dict(command=\"cmd\", requires=[\"action\"]),\n ),\n ],\n **BASE_CONFIG,\n )\n\n expected_msg = \"Unknown keys in CleanupAction : requires\"\n exception = assert_raises(\n ConfigError,\n valid_config,\n test_config,\n )\n assert_equal(expected_msg, str(exception))\n\n def test_validate_job_no_actions(self):\n job_config = dict(\n name=\"job_name\",\n node=\"localhost\",\n schedule=\"daily 00:30:00\",\n actions=[],\n )\n config_context = config_utils.ConfigContext(\n \"config\",\n [\"localhost\"],\n None,\n None,\n )\n expected_msg = \"Required non-empty list at config.Job.job_name.actions\"\n exception = assert_raises(\n ConfigError,\n valid_job,\n job_config,\n config_context,\n )\n assert_in(expected_msg, str(exception))\n\n\nclass TestValidSecretSource(TestCase):\n def test_missing_secret_name(self):\n secret_env = dict(key=\"no_secret_name\")\n\n with pytest.raises(ConfigError) as missing_exc:\n config_parse.valid_secret_source(secret_env, NullConfigContext)\n\n assert \"missing options: secret\" in str(missing_exc.value)\n\n def test_validate_job_extra_secret_env(self):\n secret_env = dict(\n secret_name=\"tron-secret-k8s-name-no--secret--name\",\n key=\"no_secret_name\",\n extra_key=\"unknown\",\n )\n with pytest.raises(ConfigError) as missing_exc:\n config_parse.valid_secret_source(secret_env, NullConfigContext)\n\n assert \"Unknown keys in SecretSource : extra_key\" in str(missing_exc.value)\n\n def test_valid_job_secret_env_success(self):\n secret_env = dict(\n secret_name=\"tron-secret-k8s-name-no--secret--name\",\n key=\"no_secret_name\",\n )\n\n expected_env = schema.ConfigSecretSource(**secret_env)\n\n built_env = config_parse.valid_secret_source(secret_env, NullConfigContext)\n assert built_env == expected_env\n\n\nclass TestNodeConfig(TestCase):\n def test_validate_node_pool(self):\n config_node_pool = valid_node_pool(\n dict(name=\"theName\", nodes=[\"node1\", \"node2\"]),\n )\n assert_equal(config_node_pool.name, \"theName\")\n assert_equal(len(config_node_pool.nodes), 2)\n\n def test_overlap_node_and_node_pools(self):\n tron_config = dict(\n nodes=[\n dict(name=\"sameName\", hostname=\"localhost\"),\n ],\n node_pools=[\n dict(name=\"sameName\", nodes=[\"sameNode\"]),\n ],\n )\n expected_msg = \"Node and NodePool names must be unique sameName\"\n exception = assert_raises(ConfigError, valid_config, tron_config)\n assert_in(expected_msg, str(exception))\n\n def test_invalid_node_name(self):\n test_config = dict(\n jobs=[\n dict(\n name=\"test_job0\",\n node=\"unknown_node\",\n schedule=\"daily 00:30:00\",\n actions=[dict(name=\"action\", command=\"cmd\")],\n ),\n ],\n **BASE_CONFIG,\n )\n\n expected_msg = \"Unknown node name unknown_node at config.jobs.Job.test_job0.node\"\n exception = assert_raises(\n ConfigError,\n valid_config,\n test_config,\n )\n assert_equal(expected_msg, str(exception))\n\n def test_invalid_nested_node_pools(self):\n test_config = dict(\n nodes=[\n dict(name=\"node0\", hostname=\"node0\"),\n dict(name=\"node1\", hostname=\"node1\"),\n ],\n node_pools=[\n dict(name=\"pool0\", nodes=[\"node1\"]),\n dict(name=\"pool1\", nodes=[\"node0\", \"pool0\"]),\n ],\n jobs=[\n dict(\n name=\"test_job0\",\n node=\"pool1\",\n schedule=\"daily 00:30:00\",\n actions=[dict(name=\"action\", command=\"cmd\")],\n ),\n ],\n )\n\n expected_msg = \"NodePool pool1 contains other NodePools: pool0\"\n exception = assert_raises(\n ConfigError,\n valid_config,\n test_config,\n )\n assert_in(expected_msg, str(exception))\n\n def test_invalid_node_pool_config(self):\n test_config = dict(\n nodes=[\n dict(name=\"node0\", hostname=\"node0\"),\n dict(name=\"node1\", hostname=\"node1\"),\n ],\n node_pools=[\n dict(name=\"pool0\", hostname=[\"node1\"]),\n dict(name=\"pool1\", nodes=[\"node0\", \"pool0\"]),\n ],\n jobs=[\n dict(\n name=\"test_job0\",\n node=\"pool1\",\n schedule=\"daily 00:30:00\",\n actions=[dict(name=\"action\", command=\"cmd\")],\n ),\n ],\n )\n\n expected_msg = \"NodePool pool0 is missing options\"\n exception = assert_raises(\n ConfigError,\n valid_config,\n test_config,\n )\n assert_in(expected_msg, str(exception))\n\n def test_invalid_named_update(self):\n test_config = dict(bozray=None)\n expected_message = \"Unknown keys in NamedConfigFragment : bozray\"\n exception = assert_raises(\n ConfigError,\n validate_fragment,\n \"foo\",\n test_config,\n )\n assert_in(expected_message, str(exception))\n\n\nclass TestValidateJobs(TestCase):\n def test_valid_jobs_success(self):\n test_config = dict(\n jobs=[\n dict(\n name=\"test_job0\",\n node=\"node0\",\n schedule=\"daily\",\n expected_runtime=\"20m\",\n actions=[\n dict(\n name=\"action\",\n command=\"command\",\n expected_runtime=\"20m\",\n ),\n dict(\n name=\"action_mesos\",\n command=\"command\",\n executor=\"mesos\",\n cpus=4,\n mem=300,\n disk=600,\n constraints=[\n dict(\n attribute=\"pool\",\n operator=\"LIKE\",\n value=\"default\",\n ),\n ],\n docker_image=\"my_container:latest\",\n docker_parameters=[\n dict(key=\"label\", value=\"labelA\"),\n dict(key=\"label\", value=\"labelB\"),\n ],\n env=dict(USER=\"batch\"),\n extra_volumes=[\n dict(\n container_path=\"/tmp\",\n host_path=\"/home/tmp\",\n mode=\"RO\",\n ),\n ],\n ),\n dict(\n name=\"test_trigger_attrs\",\n command=\"foo\",\n triggered_by=[\"foo.bar\"],\n trigger_downstreams=True,\n ),\n ],\n cleanup_action=dict(command=\"command\"),\n ),\n ],\n **BASE_CONFIG,\n )\n\n expected_jobs = {\n \"MASTER.test_job0\": make_job(\n name=\"MASTER.test_job0\",\n schedule=make_mock_schedule(),\n actions={\n \"action\": make_action(\n expected_runtime=datetime.timedelta(0, 1200),\n ),\n \"action_mesos\": make_action(\n name=\"action_mesos\",\n executor=schema.ExecutorTypes.mesos.value,\n cpus=4.0,\n mem=300.0,\n disk=600.0,\n constraints=(\n schema.ConfigConstraint(\n attribute=\"pool\",\n operator=\"LIKE\",\n value=\"default\",\n ),\n ),\n docker_image=\"my_container:latest\",\n docker_parameters=(\n schema.ConfigParameter(\n key=\"label\",\n value=\"labelA\",\n ),\n schema.ConfigParameter(\n key=\"label\",\n value=\"labelB\",\n ),\n ),\n env={\"USER\": \"batch\"},\n extra_volumes=(\n schema.ConfigVolume(\n container_path=\"/tmp\",\n host_path=\"/home/tmp\",\n mode=schema.VolumeModes.RO.value,\n ),\n ),\n expected_runtime=datetime.timedelta(hours=24),\n ),\n \"test_trigger_attrs\": make_action(\n name=\"test_trigger_attrs\",\n command=\"foo\",\n triggered_by=(\"foo.bar\",),\n trigger_downstreams=True,\n ),\n },\n expected_runtime=datetime.timedelta(0, 1200),\n ),\n }\n\n context = config_utils.ConfigContext(\n \"config\",\n [\"node0\"],\n None,\n MASTER_NAMESPACE,\n )\n config_parse.validate_jobs(test_config, context)\n assert expected_jobs == test_config[\"jobs\"]\n\n\nclass TestValidCleanupActionName(TestCase):\n def test_valid_cleanup_action_name_pass(self):\n name = valid_cleanup_action_name(CLEANUP_ACTION_NAME, None)\n assert_equal(CLEANUP_ACTION_NAME, name)\n\n def test_valid_cleanup_action_name_fail(self):\n assert_raises(\n ConfigError,\n valid_cleanup_action_name,\n \"other\",\n NullConfigContext,\n )\n\n\nclass TestValidOutputStreamDir(TestCase):\n @setup\n def setup_dir(self):\n self.dir = tempfile.mkdtemp()\n\n @teardown\n def teardown_dir(self):\n shutil.rmtree(self.dir)\n\n def test_valid_dir(self):\n path = valid_output_stream_dir(self.dir, NullConfigContext)\n assert_equal(self.dir, path)\n\n def test_missing_dir(self):\n exception = assert_raises(\n ConfigError,\n valid_output_stream_dir,\n \"bogus-dir\",\n NullConfigContext,\n )\n assert_in(\"is not a directory\", str(exception))\n\n # TODO: docker tests run as root so everything is writeable\n # def test_no_ro_dir(self):\n # os.chmod(self.dir, stat.S_IRUSR)\n # exception = assert_raises(\n # ConfigError,\n # valid_output_stream_dir, self.dir, NullConfigContext,\n # )\n # assert_in(\"is not writable\", str(exception))\n\n def test_missing_with_partial_context(self):\n dir = \"/bogus/path/does/not/exist\"\n context = config_utils.PartialConfigContext(\"path\", \"MASTER\")\n path = config_parse.valid_output_stream_dir(dir, context)\n assert_equal(path, dir)\n\n\nclass TestBuildFormatStringValidator(TestCase):\n @setup\n def setup_keys(self):\n self.context = dict.fromkeys([\"one\", \"seven\", \"stars\"])\n self.validator = build_format_string_validator(self.context)\n\n def test_validator_passes(self):\n template = \"The {one} thing I {seven} is {stars}\"\n assert self.validator(template, NullConfigContext)\n\n def test_validator_unknown_variable_error(self):\n template = \"The {one} thing I {seven} is {unknown}\"\n exception = assert_raises(\n ConfigError,\n self.validator,\n template,\n NullConfigContext,\n )\n assert_in(\"Unknown context variable\", str(exception))\n\n def test_validator_passes_with_context(self):\n template = \"The {one} thing I {seven} is {mars}\"\n context = config_utils.ConfigContext(\n None,\n None,\n {\"mars\": \"ok\"},\n None,\n )\n assert self.validator(template, context) == template\n\n def test_validator_valid_string_without_no_percent_escape(self):\n template = \"The {one} {seven} thing is {mars} --year %Y\"\n context = config_utils.ConfigContext(\n path=None,\n nodes=None,\n command_context={\"mars\": \"ok\"},\n namespace=None,\n )\n assert self.validator(template, context)\n\n\nclass TestValidateConfigMapping(TestCase):\n config = dict(**BASE_CONFIG, command_context=dict(some_var=\"The string\"))\n\n def test_validate_config_mapping_missing_master(self):\n config_mapping = {\"other\": mock.Mock()}\n seq = config_parse.validate_config_mapping(config_mapping)\n exception = assert_raises(ConfigError, list, seq)\n assert_in(\"requires a MASTER namespace\", str(exception))\n\n def test_validate_config_mapping(self):\n master_config = self.config\n other_config = TestNamedConfig.config\n config_mapping = {\n \"other\": other_config,\n MASTER_NAMESPACE: master_config,\n }\n result = list(config_parse.validate_config_mapping(config_mapping))\n assert_equal(len(result), 2)\n assert_equal(result[0][0], MASTER_NAMESPACE)\n assert_equal(result[1][0], \"other\")\n\n\nclass TestConfigContainer(TestCase):\n config = BASE_CONFIG\n\n @setup\n def setup_container(self):\n other_config = TestNamedConfig.config\n self.config_mapping = {\n MASTER_NAMESPACE: valid_config(self.config),\n \"other\": validate_fragment(\"other\", other_config),\n }\n self.container = config_parse.ConfigContainer(self.config_mapping)\n\n def test_create(self):\n config_mapping = {\n MASTER_NAMESPACE: self.config,\n \"other\": TestNamedConfig.config,\n }\n\n container = config_parse.ConfigContainer.create(config_mapping)\n assert_equal(set(container.configs.keys()), {\"MASTER\", \"other\"})\n\n def test_create_missing_master(self):\n config_mapping = {\"other\": mock.Mock()}\n assert_raises(\n ConfigError,\n config_parse.ConfigContainer.create,\n config_mapping,\n )\n\n def test_get_job_names(self):\n job_names = self.container.get_job_names()\n expected = [\n \"test_job1\",\n \"test_job0\",\n \"test_job_actions_dict\",\n \"test_job2\",\n \"test_job4\",\n \"test_job_mesos\",\n \"test_job_k8s\",\n ]\n assert_equal(set(job_names), set(expected))\n\n def test_get_jobs(self):\n expected = [\n \"test_job1\",\n \"test_job0\",\n \"test_job_actions_dict\",\n \"test_job2\",\n \"test_job4\",\n \"test_job_mesos\",\n \"test_job_k8s\",\n ]\n assert_equal(set(expected), set(self.container.get_jobs().keys()))\n\n def test_get_node_names(self):\n node_names = self.container.get_node_names()\n expected = {\"node0\", \"node1\", \"NodePool\"}\n assert_equal(node_names, expected)\n\n\nclass TestValidateSSHOptions(TestCase):\n @setup\n def setup_context(self):\n self.context = config_utils.NullConfigContext\n self.config = {\"agent\": True, \"identities\": []}\n\n @mock.patch.dict(\"tron.config.config_parse.os.environ\")\n def test_post_validation_failed(self):\n if \"SSH_AUTH_SOCK\" in os.environ:\n del os.environ[\"SSH_AUTH_SOCK\"]\n assert_raises(\n ConfigError,\n config_parse.valid_ssh_options.validate,\n self.config,\n self.context,\n )\n\n @mock.patch.dict(\"tron.config.config_parse.os.environ\")\n def test_post_validation_success(self):\n os.environ[\"SSH_AUTH_SOCK\"] = \"something\"\n config = config_parse.valid_ssh_options.validate(\n self.config,\n self.context,\n )\n assert_equal(config.agent, True)\n\n\nclass TestValidateIdentityFile(TestCase):\n @setup\n def setup_context(self):\n self.context = config_utils.NullConfigContext\n self.private_file = tempfile.NamedTemporaryFile()\n\n def test_valid_identity_file_missing_private_key(self):\n exception = assert_raises(\n ConfigError,\n config_parse.valid_identity_file,\n \"/file/not/exist\",\n self.context,\n )\n assert_in(\"Private key file\", str(exception))\n\n def test_valid_identity_files_missing_public_key(self):\n filename = self.private_file.name\n exception = assert_raises(\n ConfigError,\n config_parse.valid_identity_file,\n filename,\n self.context,\n )\n assert_in(\"Public key file\", str(exception))\n\n def test_valid_identity_files_valid(self):\n filename = self.private_file.name\n fh_private = open(filename + \".pub\", \"w\")\n try:\n config = config_parse.valid_identity_file(filename, self.context)\n finally:\n fh_private.close()\n os.unlink(fh_private.name)\n assert_equal(config, filename)\n\n def test_valid_identity_files_missing_with_partial_context(self):\n path = \"/bogus/file/does/not/exist\"\n context = config_utils.PartialConfigContext(\"path\", \"MASTER\")\n file_path = config_parse.valid_identity_file(path, context)\n assert_equal(path, file_path)\n\n\nclass TestValidKnownHostsFile(TestCase):\n @setup\n def setup_context(self):\n self.context = config_utils.NullConfigContext\n self.known_hosts_file = tempfile.NamedTemporaryFile()\n\n def test_valid_known_hosts_file_exists(self):\n filename = config_parse.valid_known_hosts_file(\n self.known_hosts_file.name,\n self.context,\n )\n assert_equal(filename, self.known_hosts_file.name)\n\n def test_valid_known_hosts_file_missing(self):\n exception = assert_raises(\n ConfigError,\n config_parse.valid_known_hosts_file,\n \"/bogus/path\",\n self.context,\n )\n assert_in(\"Known hosts file /bogus/path\", str(exception))\n\n def test_valid_known_hosts_file_missing_partial_context(self):\n context = config_utils.PartialConfigContext\n expected = \"/bogus/does/not/exist\"\n filename = config_parse.valid_known_hosts_file(\n expected,\n context,\n )\n assert_equal(filename, expected)\n\n\nclass TestValidateVolume(TestCase):\n @setup\n def setup_context(self):\n self.context = config_utils.NullConfigContext\n\n def test_missing_container_path(self):\n config = {\n \"container_path_typo\": \"/nail/srv\",\n \"host_path\": \"/tmp\",\n \"mode\": \"RO\",\n }\n assert_raises(\n ConfigError,\n config_parse.valid_volume.validate,\n config,\n self.context,\n )\n\n def test_missing_host_path(self):\n config = {\n \"container_path\": \"/nail/srv\",\n \"hostPath\": \"/tmp\",\n \"mode\": \"RO\",\n }\n assert_raises(\n ConfigError,\n config_parse.valid_volume.validate,\n config,\n self.context,\n )\n\n def test_invalid_mode(self):\n config = {\n \"container_path\": \"/nail/srv\",\n \"host_path\": \"/tmp\",\n \"mode\": \"RA\",\n }\n assert_raises(\n ConfigError,\n config_parse.valid_volume.validate,\n config,\n self.context,\n )\n\n def test_valid(self):\n config = {\n \"container_path\": \"/nail/srv\",\n \"host_path\": \"/tmp\",\n \"mode\": schema.VolumeModes.RO.value,\n }\n assert_equal(\n schema.ConfigVolume(**config),\n config_parse.valid_volume.validate(config, self.context),\n )\n\n def test_mesos_default_volumes(self):\n mesos_options = {\"master_address\": \"mesos_master\"}\n mesos_options[\"default_volumes\"] = [\n {\n \"container_path\": \"/nail/srv\",\n \"host_path\": \"/tmp\",\n \"mode\": \"RO\",\n },\n {\n \"container_path\": \"/nail/srv\",\n \"host_path\": \"/tmp\",\n \"mode\": \"invalid\",\n },\n ]\n\n with pytest.raises(ConfigError):\n config_parse.valid_mesos_options.validate(mesos_options, self.context)\n\n # After we fix the error, expect error to go away.\n mesos_options[\"default_volumes\"][1][\"mode\"] = \"RW\"\n assert config_parse.valid_mesos_options.validate(\n mesos_options,\n self.context,\n )\n\n def test_k8s_default_volumes(self):\n k8s_options = {\"kubeconfig_path\": \"some_path\"}\n k8s_options[\"default_volumes\"] = [\n {\n \"container_path\": \"/nail/srv\",\n \"host_path\": \"/tmp\",\n \"mode\": \"RO\",\n },\n {\n \"container_path\": \"/nail/srv\",\n \"host_path\": \"/tmp\",\n \"mode\": \"invalid\",\n },\n ]\n\n with pytest.raises(ConfigError):\n config_parse.valid_kubernetes_options.validate(k8s_options, self.context)\n\n # After we fix the error, expect error to go away.\n k8s_options[\"default_volumes\"][1][\"mode\"] = \"RW\"\n assert config_parse.valid_kubernetes_options.validate(\n k8s_options,\n self.context,\n )\n\n\nclass TestValidPermissionMode:\n @pytest.mark.parametrize(\n (\"permission\", \"normalized\"),\n [(\"777\", \"777\"), (\"0\", \"0\"), (\"0000\", \"0000\"), (\"0123\", \"0123\"), (0, \"0\"), (7777, \"7777\")],\n )\n def test_valid_permissions(self, permission, normalized):\n result = config_parse.valid_permission_mode(permission, NullConfigContext)\n assert result == normalized\n\n @pytest.mark.parametrize(\"permission\", [\"778\", \"é\", -1, \"\", {}, [], ()])\n def test_invalid_permissions(self, permission):\n with pytest.raises(ConfigError):\n config_parse.valid_permission_mode(permission, NullConfigContext)\n\n\nclass TestValidSecretVolumeItem:\n @pytest.mark.parametrize(\n \"config\",\n [\n {\"path\": \"abc\"},\n {\n \"key\": \"abc\",\n },\n {\n \"key\": \"abc\",\n \"path\": \"abc\",\n \"extra_key\": None,\n },\n {\"key\": \"abc\", \"path\": \"abc\", \"mode\": \"a\"},\n ],\n )\n def test_invalid(self, config):\n with pytest.raises(ConfigError):\n config_parse.valid_secret_volume_item(config, NullConfigContext)\n\n @pytest.mark.parametrize(\n \"config\",\n [{\"key\": \"abc\", \"path\": \"abc\"}, {\"key\": \"abc\", \"path\": \"abc\", \"mode\": \"777\"}],\n )\n def test_valid_job_secret_volume_success(self, config):\n config_parse.valid_secret_volume_item(config, NullConfigContext)\n\n @pytest.mark.parametrize(\n \"item_config, default_mode, expected\",\n [\n (\n # Item inherits volume default_mode\n {\"key\": \"s1\", \"path\": \"p1\"},\n \"0755\",\n \"0755\",\n ),\n (\n # Item explicit mode overrides volume default_mode\n {\"key\": \"s2\", \"path\": \"p2\", \"mode\": \"0600\"},\n \"0755\",\n \"0600\",\n ),\n (\n # Item inherits class default for volume default_mode\n {\"key\": \"s3\", \"path\": \"p3\"},\n None,\n \"0644\",\n ),\n ],\n )\n def test_item_mode_propagation_and_override(self, item_config, default_mode, expected):\n input_config = {\n \"secret_volume_name\": \"test_vol\",\n \"secret_name\": item_config[\"key\"],\n \"container_path\": \"/secrets\",\n \"items\": [item_config],\n }\n if default_mode is not None:\n input_config[\"default_mode\"] = default_mode\n\n context = config_utils.NullConfigContext\n validated_volume_obj = config_parse.valid_secret_volume(input_config, context)\n\n assert validated_volume_obj.items is not None\n assert len(validated_volume_obj.items) == 1\n\n actual_item_mode = validated_volume_obj.items[0].mode\n assert actual_item_mode == expected\n\n def test_volume_when_items_key_is_omitted(self):\n input_config = {\n \"secret_volume_name\": \"vol_no_items\",\n \"secret_name\": \"s_no_items\",\n \"container_path\": \"/secrets\",\n \"default_mode\": \"0400\",\n }\n context = config_utils.NullConfigContext\n validated_volume_obj = config_parse.valid_secret_volume(input_config, context)\n\n assert validated_volume_obj.default_mode == \"0400\"\n assert validated_volume_obj.items is None\n\n def test_volume_when_items_is_empty(self):\n input_config = {\n \"secret_volume_name\": \"vol_empty_items\",\n \"secret_name\": \"s_empty_items\",\n \"container_path\": \"/secrets\",\n \"default_mode\": \"0700\",\n \"items\": [],\n }\n context = config_utils.NullConfigContext\n validated_volume_obj = config_parse.valid_secret_volume(input_config, context)\n\n assert validated_volume_obj.default_mode == \"0700\"\n assert validated_volume_obj.items is not None\n assert isinstance(validated_volume_obj.items, tuple)\n assert len(validated_volume_obj.items) == 0\n\n\nclass TestValidSecretVolume:\n @pytest.mark.parametrize(\n \"config\",\n [\n dict(\n secret_volume_name=\"abc\",\n secret_name=\"secret1\",\n container_path=\"/b/c\",\n default_mode=\"0644\",\n items=[\n dict(key=\"secret1\", path=\"abcd\", mode=\"7778\"),\n ],\n ),\n dict(\n secret_volume_name=\"abc\",\n container_path=\"/b/c\",\n default_mode=\"0644\",\n items=[\n dict(key=\"secret1\", path=\"abcd\", mode=\"7777\"),\n ],\n ),\n dict(\n secret_volume_name=\"abc\",\n secret_name=\"secret1\",\n container_path=123,\n ),\n dict(\n secret_volume_name=\"abc\",\n secret_name=\"secret1\",\n container_path=\"/b/c\",\n items=[dict(key=\"secret1\", path=\"abcd\", mode=\"7777\"), dict(key=\"secret1\", path=\"abcde\", mode=\"7777\")],\n ),\n ],\n )\n def test_invalid(self, config):\n with pytest.raises(ConfigError):\n config_parse.valid_secret_volume(config, NullConfigContext)\n\n def test_wrong_item_key(self):\n config = dict(\n secret_volume_name=\"abc\",\n secret_name=\"secret1\",\n container_path=\"/b/c\",\n items=[\n dict(key=\"secret2\", path=\"abc\"),\n ],\n )\n with pytest.raises(ConfigError):\n config_parse.valid_secret_volume(config, NullConfigContext)\n\n @pytest.mark.parametrize(\n \"config\",\n [\n dict(\n secret_volume_name=\"abc\",\n secret_name=\"secret1\",\n container_path=\"/b/c\",\n default_mode=\"0644\",\n items=[\n dict(key=\"secret1\", path=\"abc\"),\n ],\n ),\n dict(\n secret_volume_name=\"abc\",\n secret_name=\"secret1\",\n container_path=\"/b/c\",\n items=[],\n ),\n dict(\n secret_volume_name=\"abc\",\n secret_name=\"secret1\",\n container_path=\"/b/c\",\n ),\n ],\n )\n def test_valid(self, config):\n config_parse.valid_secret_volume(config, NullConfigContext)\n\n\nclass TestValidMasterAddress:\n @pytest.fixture\n def context(self):\n return config_utils.NullConfigContext\n\n @pytest.mark.parametrize(\n \"url\",\n [\n \"http://blah.com\",\n \"http://blah.com/\",\n \"blah.com\",\n \"blah.com/\",\n ],\n )\n def test_valid(self, url, context):\n normalized = \"http://blah.com\"\n result = config_parse.valid_master_address(url, context)\n assert result == normalized\n\n @pytest.mark.parametrize(\n \"url\",\n [\n \"https://blah.com\",\n \"http://blah.com/something\",\n \"blah.com/other\",\n \"http://\",\n \"blah.com?a=1\",\n ],\n )\n def test_invalid(self, url, context):\n with pytest.raises(ConfigError):\n config_parse.valid_master_address(url, context)\n\n\nclass TestValidKubeconfigPaths:\n @setup\n def setup_context(self):\n self.context = config_utils.NullConfigContext\n\n @pytest.mark.parametrize(\n \"kubeconfig_path,watcher_kubeconfig_paths\",\n [(\"/some/kubeconfig.conf\", []), (\"/another/kube/config\", [\"a_watcher_kubeconfig\"])],\n )\n def test_valid(self, kubeconfig_path, watcher_kubeconfig_paths):\n k8s_options = {\n \"enabled\": True,\n \"kubeconfig_path\": kubeconfig_path,\n \"watcher_kubeconfig_paths\": watcher_kubeconfig_paths,\n }\n assert config_parse.valid_kubernetes_options.validate(k8s_options, self.context)\n\n @pytest.mark.parametrize(\n \"kubeconfig_path,watcher_kubeconfig_paths\",\n [\n ([\"/a/kubeconfig/in/a/list\"], [\"/a/valid/kubeconfig\"]),\n (None, []),\n (\"/some/kubeconfig.conf\", \"/not/a/list/kubeconfig\"),\n ],\n )\n def test_invalid(self, kubeconfig_path, watcher_kubeconfig_paths):\n k8s_options = {\n \"enabled\": True,\n \"kubeconfig_path\": kubeconfig_path,\n \"watcher_kubeconfig_paths\": watcher_kubeconfig_paths,\n }\n with pytest.raises(ConfigError):\n config_parse.valid_kubernetes_options.validate(k8s_options, self.context)\n\n def test_nonretry(self):\n k8s_options = {\n \"enabled\": True,\n \"kubeconfig_path\": \"/some/valid/path\",\n \"watcher_kubeconfig_paths\": [],\n \"non_retryable_exit_codes\": 1,\n }\n with pytest.raises(ConfigError):\n config_parse.valid_kubernetes_options.validate(k8s_options, self.context)\n\n k8s_options[\"non_retryable_exit_codes\"] = [-12, 1]\n\n assert config_parse.valid_kubernetes_options.validate(k8s_options, self.context)\n\n\nclass TestValidateStatePersistenceDefaults(TestCase):\n def test_post_validation_sees_defaults_for_omitted_keys(self):\n input_config = {\n \"store_type\": \"dynamodb\",\n \"name\": \"test_state\",\n \"table_name\": \"test_table\",\n \"dynamodb_region\": \"us-west-2\",\n \"buffer_size\": 5,\n # max_transact_write_items\n }\n\n original_post_validation = config_parse.ValidateStatePersistence.post_validation\n post_validation_args = {}\n\n def mock_post_validation_side_effect(self_validator, output_dict, config_context):\n post_validation_args[\"max_transact_write_items\"] = output_dict.get(\"max_transact_write_items\")\n post_validation_args[\"buffer_size\"] = output_dict.get(\"buffer_size\")\n\n return original_post_validation(self_validator, output_dict, config_context)\n\n with mock.patch.object(\n config_parse.ValidateStatePersistence,\n \"post_validation\",\n side_effect=mock_post_validation_side_effect,\n autospec=True,\n ) as mock_method:\n validator = config_parse.ValidateStatePersistence()\n context = config_utils.NullConfigContext\n validated_config = validator(input_config, context)\n mock_method.assert_called_once()\n\n assert post_validation_args.get(\"max_transact_write_items\") == 8\n assert post_validation_args.get(\"buffer_size\") == 5\n\n assert validated_config.store_type == \"dynamodb\"\n assert validated_config.name == \"test_state\"\n assert validated_config.table_name == \"test_table\"\n assert validated_config.dynamodb_region == \"us-west-2\"\n assert validated_config.buffer_size == 5\n assert validated_config.max_transact_write_items == 8\n\n def test_post_validation_sees_provided_values(self):\n input_config = {\n \"store_type\": \"dynamodb\",\n \"name\": \"test_state\",\n \"table_name\": \"test_table\",\n \"dynamodb_region\": \"us-west-2\",\n \"buffer_size\": 5,\n \"max_transact_write_items\": 25,\n }\n\n original_post_validation = config_parse.ValidateStatePersistence.post_validation\n post_validation_args = {}\n\n def mock_post_validation_side_effect(self_validator, output_dict, config_context):\n post_validation_args[\"max_transact_write_items\"] = output_dict.get(\"max_transact_write_items\")\n return original_post_validation(self_validator, output_dict, config_context)\n\n with mock.patch.object(\n config_parse.ValidateStatePersistence,\n \"post_validation\",\n side_effect=mock_post_validation_side_effect,\n autospec=True,\n ) as mock_method:\n validator = config_parse.ValidateStatePersistence()\n context = config_utils.NullConfigContext\n validated_config = validator(input_config, context)\n mock_method.assert_called_once()\n\n assert post_validation_args.get(\"max_transact_write_items\") == 25\n\n assert validated_config.max_transact_write_items == 25\n assert validated_config.store_type == \"dynamodb\"\n assert validated_config.name == \"test_state\"\n assert validated_config.table_name == \"test_table\"\n assert validated_config.dynamodb_region == \"us-west-2\"\n assert validated_config.buffer_size == 5\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/config/config_utils_test.py", "content": "import datetime\nfrom unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_in\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tests.assertions import assert_raises\nfrom tron.config import config_utils\nfrom tron.config import ConfigError\nfrom tron.config import schema\nfrom tron.config.config_utils import build_list_of_type_validator\nfrom tron.config.config_utils import ConfigContext\nfrom tron.config.config_utils import valid_identifier\n\n\nclass TestUniqueNameDict(TestCase):\n @setup\n def setup_dict(self):\n self.msg = \"The key %s was there.\"\n self.dict = config_utils.UniqueNameDict(self.msg)\n\n def test_set_item_no_conflict(self):\n self.dict[\"a\"] = \"something\"\n assert_in(\"a\", self.dict)\n\n def test_set_item_conflict(self):\n self.dict[\"a\"] = \"something\"\n assert_raises(ConfigError, self.dict.__setitem__, \"a\", \"next_thing\")\n\n\nclass TestValidatorIdentifier(TestCase):\n def test_valid_identifier_too_long(self):\n assert_raises(ConfigError, valid_identifier, \"a\" * 256, mock.Mock())\n\n def test_valid_identifier(self):\n name = \"avalidname\"\n assert_equal(name, valid_identifier(name, mock.Mock()))\n\n def test_valid_identifier_invalid_character(self):\n for name in [\"invalid space\", \"*name\", \"1numberstarted\", 123, \"\"]:\n assert_raises(ConfigError, valid_identifier, name, mock.Mock())\n\n\nclass TestBuildListOfTypeValidator(TestCase):\n @setup\n def setup_validator(self):\n self.item_validator = mock.Mock()\n self.validator = build_list_of_type_validator(self.item_validator)\n\n def test_validator_passes(self):\n items, context = [\"one\", \"two\"], mock.create_autospec(ConfigContext)\n self.validator(items, context)\n expected = [mock.call(item, context) for item in items]\n assert_equal(self.item_validator.mock_calls, expected)\n\n def test_validator_fails(self):\n self.item_validator.side_effect = ConfigError\n items, context = [\"one\", \"two\"], mock.create_autospec(ConfigContext)\n assert_raises(ConfigError, self.validator, items, context)\n\n\nclass TestBuildEnumValidator(TestCase):\n @setup\n def setup_enum_validator(self):\n self.enum = dict(a=1, b=2)\n self.validator = config_utils.build_enum_validator(self.enum)\n self.context = config_utils.NullConfigContext\n\n def test_validate(self):\n assert_equal(self.validator(\"a\", self.context), \"a\")\n assert_equal(self.validator(\"b\", self.context), \"b\")\n\n def test_invalid(self):\n exception = assert_raises(\n ConfigError,\n self.validator,\n \"c\",\n self.context,\n )\n assert_in(\n \"Value at is not in %s: \" % str(set(self.enum)),\n str(exception),\n )\n\n\nclass TestValidTime(TestCase):\n @setup\n def setup_config(self):\n self.context = config_utils.NullConfigContext\n\n def test_valid_time(self):\n time_spec = config_utils.valid_time(\"14:32\", self.context)\n assert_equal(time_spec.hour, 14)\n assert_equal(time_spec.minute, 32)\n assert_equal(time_spec.second, 0)\n\n def test_valid_time_with_seconds(self):\n time_spec = config_utils.valid_time(\"14:32:12\", self.context)\n assert_equal(time_spec.hour, 14)\n assert_equal(time_spec.minute, 32)\n assert_equal(time_spec.second, 12)\n\n def test_valid_time_invalid(self):\n assert_raises(\n ConfigError,\n config_utils.valid_time,\n \"14:32:12:34\",\n self.context,\n )\n assert_raises(ConfigError, config_utils.valid_time, None, self.context)\n\n\nclass TestValidTimeDelta(TestCase):\n @setup\n def setup_config(self):\n self.context = config_utils.NullConfigContext\n\n def test_valid_time_delta_invalid(self):\n exception = assert_raises(\n ConfigError,\n config_utils.valid_time_delta,\n \"no time\",\n self.context,\n )\n assert_in(\"not a valid time delta: no time\", str(exception))\n\n def test_valid_time_delta_valid_seconds(self):\n for jitter in [\" 82s \", \"82 s\", \"82 sec\", \"82seconds \"]:\n delta = datetime.timedelta(seconds=82)\n assert_equal(\n delta,\n config_utils.valid_time_delta(\n jitter,\n self.context,\n ),\n )\n\n def test_valid_time_delta_valid_minutes(self):\n for jitter in [\"10m\", \"10 m\", \"10 min\", \" 10minutes\"]:\n delta = datetime.timedelta(seconds=600)\n assert_equal(\n delta,\n config_utils.valid_time_delta(\n jitter,\n self.context,\n ),\n )\n\n def test_valid_time_delta_invalid_unit(self):\n for jitter in [\"1 year\", \"3 mo\", \"3 months\"]:\n assert_raises(\n ConfigError,\n config_utils.valid_time_delta,\n jitter,\n self.context,\n )\n\n\nclass TestConfigContext(TestCase):\n def test_build_config_context(self):\n path, nodes, namespace = \"path\", {1, 2, 3}, \"namespace\"\n command_context = mock.MagicMock()\n parent_context = config_utils.ConfigContext(\n path,\n nodes,\n command_context,\n namespace,\n )\n\n child = parent_context.build_child_context(\"child\")\n assert_equal(child.path, \"%s.child\" % path)\n assert_equal(child.nodes, nodes)\n assert_equal(child.namespace, namespace)\n assert_equal(child.command_context, command_context)\n assert not child.partial\n\n\nStubConfigObject = schema.config_object_factory(\n \"StubConfigObject\",\n [\"req1\", \"req2\"],\n [\"opt1\", \"opt2\"],\n)\n\n\nclass StubValidator(config_utils.Validator):\n config_class = StubConfigObject\n\n\nclass TestValidator(TestCase):\n @setup\n def setup_validator(self):\n self.validator = StubValidator()\n\n def test_validate_with_none(self):\n expected_msg = \"A StubObject is required\"\n exception = assert_raises(\n ConfigError,\n self.validator.validate,\n None,\n config_utils.NullConfigContext,\n )\n assert_in(expected_msg, str(exception))\n\n def test_validate_optional_with_none(self):\n self.validator.optional = True\n config = self.validator.validate(None, config_utils.NullConfigContext)\n assert_equal(config, None)\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/config/manager_test.py", "content": "import os\nimport shutil\nimport tempfile\nfrom unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import teardown\nfrom testifycompat import TestCase\nfrom tests.assertions import assert_raises\nfrom tests.testingutils import autospec_method\nfrom tron import yaml\nfrom tron.config import ConfigError\nfrom tron.config import manager\nfrom tron.config import schema\n\n\nclass TestFromString(TestCase):\n def test_from_string_valid(self):\n content = \"{'one': 'thing', 'another': 'thing'}\\n\"\n actual = manager.from_string(content)\n expected = {\"one\": \"thing\", \"another\": \"thing\"}\n assert_equal(actual, expected)\n\n def test_from_string_invalid(self):\n content = \"{} asdf\"\n assert_raises(ConfigError, manager.from_string, content)\n\n\nclass TestReadWrite(TestCase):\n @setup\n def setup_tempfile(self):\n self.filename = tempfile.NamedTemporaryFile().name\n\n @teardown\n def teardown_tempfile(self):\n os.unlink(self.filename)\n\n def test_read_write(self):\n content = {\"one\": \"stars\", \"two\": \"beers\"}\n manager.write(self.filename, content)\n actual = manager.read(self.filename)\n assert_equal(content, actual)\n\n def test_read_raw_write_raw(self):\n content = \"Some string\"\n manager.write_raw(self.filename, content)\n actual = manager.read_raw(self.filename)\n assert_equal(content, actual)\n\n\nclass TestManifestFile(TestCase):\n @setup\n def setup_manifest(self):\n self.temp_dir = tempfile.mkdtemp()\n self.manifest = manager.ManifestFile(self.temp_dir)\n self.manifest.create()\n\n @teardown\n def teardown_dir(self):\n shutil.rmtree(self.temp_dir)\n\n @mock.patch(\"tron.config.manager.os.path\", autospec=True)\n @mock.patch(\"tron.config.manager.write\", autospec=True)\n def test_create_exists(self, mock_write, mock_os):\n mock_os.isfile.return_value = True\n self.manifest.create()\n assert not mock_write.call_count\n\n def test_create(self):\n assert_equal(manager.read(self.manifest.filename), {})\n\n def test_add(self):\n self.manifest.add(\"zing\", \"zing.yaml\")\n expected = {\"zing\": \"zing.yaml\"}\n assert_equal(manager.read(self.manifest.filename), expected)\n\n def test_delete(self):\n current = {\n \"one\": \"a.yaml\",\n \"two\": \"b.yaml\",\n }\n manager.write(self.manifest.filename, current)\n self.manifest.delete(\"one\")\n expected = {\"two\": \"b.yaml\"}\n assert_equal(manager.read(self.manifest.filename), expected)\n\n def test_get_file_mapping(self):\n file_mapping = {\n \"one\": \"a.yaml\",\n \"two\": \"b.yaml\",\n }\n manager.write(self.manifest.filename, file_mapping)\n assert_equal(self.manifest.get_file_mapping(), file_mapping)\n\n\nclass TestConfigManager(TestCase):\n\n content = {\"one\": \"stars\", \"two\": \"other\"}\n raw_content = \"{'one': 'stars', 'two': 'other'}\\n\"\n\n @setup\n def setup_config_manager(self):\n self.temp_dir = tempfile.mkdtemp()\n self.manager = manager.ConfigManager(self.temp_dir)\n self.manifest = mock.create_autospec(manager.ManifestFile)\n self.manager.manifest = self.manifest\n\n @teardown\n def teardown_dir(self):\n shutil.rmtree(self.temp_dir)\n\n def test_build_file_path(self):\n path = self.manager.build_file_path(\"what\")\n assert_equal(path, os.path.join(self.temp_dir, \"what.yaml\"))\n\n def test_build_file_path_with_invalid_chars(self):\n path = self.manager.build_file_path(\"/etc/passwd\")\n assert_equal(path, os.path.join(self.temp_dir, \"_etc_passwd.yaml\"))\n path = self.manager.build_file_path(\"../../etc/passwd\")\n assert_equal(\n path,\n os.path.join(\n self.temp_dir,\n \"______etc_passwd.yaml\",\n ),\n )\n\n def test_read_raw_config(self):\n name = \"name\"\n path = os.path.join(self.temp_dir, name)\n manager.write(path, self.content)\n self.manifest.get_file_name.return_value = path\n config = self.manager.read_raw_config(name)\n assert_equal(config, yaml.dump(self.content))\n\n def test_write_config(self):\n name = \"filename\"\n path = self.manager.build_file_path(name)\n self.manifest.get_file_name.return_value = path\n autospec_method(self.manager.validate_with_fragment)\n self.manager.write_config(name, self.raw_content)\n assert_equal(manager.read(path), self.content)\n self.manifest.get_file_name.assert_called_with(name)\n assert not self.manifest.add.call_count\n self.manager.validate_with_fragment.assert_called_with(\n name,\n self.content,\n should_validate_missing_dependency=False,\n )\n\n def test_write_config_new_name(self):\n name = \"filename2\"\n path = self.manager.build_file_path(name)\n self.manifest.get_file_name.return_value = None\n autospec_method(self.manager.validate_with_fragment)\n self.manager.write_config(name, self.raw_content)\n assert_equal(manager.read(path), self.content)\n self.manifest.get_file_name.assert_called_with(name)\n self.manifest.add.assert_called_with(name, path)\n\n @mock.patch(\"os.remove\", autospec=True)\n def test_delete_config(self, mock_remove):\n name = \"namespace\"\n path = \"namespace.yaml\"\n self.manifest.get_file_name.return_value = path\n self.manager.delete_config(name)\n self.manifest.delete.assert_called_with(name)\n mock_remove.assert_called_with(path)\n\n @mock.patch(\"os.remove\", autospec=True)\n def test_delete_missing_namespace(self, mock_remove):\n name = \"namespace\"\n self.manifest.get_file_name.return_value = None\n self.manager.delete_config(name)\n assert_equal(mock_remove.call_count, 0)\n\n @mock.patch(\n \"tron.config.manager.JobGraph\",\n autospec=True,\n )\n @mock.patch(\n \"tron.config.manager.config_parse.ConfigContainer\",\n autospec=True,\n )\n def test_validate_with_fragment(self, mock_config_container, mock_job_graph):\n name = \"the_name\"\n name_mapping = {\"something\": \"content\", name: \"old_content\"}\n autospec_method(self.manager.get_config_name_mapping)\n self.manager.get_config_name_mapping.return_value = name_mapping\n self.manager.validate_with_fragment(name, self.content)\n expected_mapping = dict(name_mapping)\n expected_mapping[name] = self.content\n mock_config_container.create.assert_called_with(expected_mapping)\n mock_job_graph.assert_called_once_with(\n mock_config_container.create.return_value,\n should_validate_missing_dependency=True,\n )\n\n @mock.patch(\"tron.config.manager.read\", autospec=True)\n @mock.patch(\n \"tron.config.manager.config_parse.ConfigContainer\",\n autospec=True,\n )\n def test_load(self, mock_config_container, mock_read):\n content_items = self.content.items()\n self.manifest.get_file_mapping().return_value = content_items\n container = self.manager.load()\n self.manifest.get_file_mapping.assert_called_with()\n assert_equal(container, mock_config_container.create.return_value)\n\n expected = {name: call.return_value for ((name, _), call) in zip(content_items, mock_read.mock_calls)}\n mock_config_container.create.assert_called_with(expected)\n\n def test_get_hash_default(self):\n self.manifest.__contains__.return_value = False\n hash_digest = self.manager.get_hash(\"name\")\n assert_equal(hash_digest, self.manager.DEFAULT_HASH)\n\n def test_get_hash(self):\n content = \"OkOkOk\"\n autospec_method(self.manager.read_raw_config, return_value=content)\n self.manifest.__contains__.return_value = True\n hash_digest = self.manager.get_hash(\"name\")\n assert_equal(hash_digest, manager.hash_digest(content))\n\n\nclass TestCreateNewConfig(TestCase):\n @mock.patch(\"tron.config.manager.os.makedirs\", autospec=True)\n @mock.patch(\"tron.config.manager.ManifestFile\", autospec=True)\n @mock.patch(\"tron.config.manager.write_raw\", autospec=True)\n def test_create_new_config(self, mock_write, mock_manifest, mock_makedirs):\n path, master_content = \"/bogus/path/\", mock.Mock()\n filename = \"/bogus/path/MASTER.yaml\"\n manifest = mock_manifest.return_value\n manifest.get_file_name.return_value = None\n\n manager.create_new_config(path, master_content)\n mock_makedirs.assert_called_with(path)\n mock_write.assert_called_with(filename, master_content)\n manifest.create.assert_called_with()\n manifest.add.assert_called_with(schema.MASTER_NAMESPACE, filename)\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/config/schedule_parse_test.py", "content": "import datetime\nfrom unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_raises\nfrom testifycompat import run\nfrom testifycompat import TestCase\nfrom tron.config import config_utils\nfrom tron.config import ConfigError\nfrom tron.config import schedule_parse\n\n\nclass TestPadSequence(TestCase):\n def test_pad_sequence_short(self):\n expected = [0, 1, 2, 3, None, None]\n assert_equal(schedule_parse.pad_sequence(range(4), 6), expected)\n\n def test_pad_sequence_long(self):\n expected = [0, 1, 2, 3]\n assert_equal(schedule_parse.pad_sequence(range(6), 4), expected)\n\n def test_pad_sequence_exact(self):\n expected = [0, 1, 2, 3]\n assert_equal(schedule_parse.pad_sequence(range(4), 4), expected)\n\n def test_pad_sequence_empty(self):\n expected = [\"a\", \"a\"]\n assert_equal(schedule_parse.pad_sequence([], 2, \"a\"), expected)\n\n def test_pad_negative_size(self):\n assert_equal(schedule_parse.pad_sequence([], -2, \"a\"), [])\n\n\nclass TestScheduleConfigFromString(TestCase):\n @mock.patch(\n \"tron.config.schedule_parse.parse_groc_expression\",\n autospec=True,\n )\n def test_groc_config(self, mock_parse_groc):\n schedule = \"every Mon,Wed at 12:00\"\n context = config_utils.NullConfigContext\n config = schedule_parse.schedule_config_from_string(schedule, context)\n assert_equal(config, mock_parse_groc.return_value)\n generic_config = schedule_parse.ConfigGenericSchedule(\n \"groc daily\",\n schedule,\n None,\n )\n mock_parse_groc.assert_called_with(generic_config, context)\n\n\nclass TestValidScheduler(TestCase):\n @mock.patch(\"tron.config.schedule_parse.schedulers\", autospec=True)\n def assert_validation(self, schedule, expected, mock_schedulers):\n context = config_utils.NullConfigContext\n config = schedule_parse.valid_schedule(schedule, context)\n mock_schedulers.__getitem__.assert_called_with(\"cron\")\n func = mock_schedulers.__getitem__.return_value\n assert_equal(config, func.return_value)\n func.assert_called_with(expected, context)\n\n def test_cron_from_dict(self):\n schedule = {\"type\": \"cron\", \"value\": \"* * * * *\"}\n config = schedule_parse.ConfigGenericSchedule(\n \"cron\",\n schedule[\"value\"],\n datetime.timedelta(),\n )\n self.assert_validation(schedule, config)\n\n def test_cron_from_dict_with_jitter(self):\n schedule = {\"type\": \"cron\", \"value\": \"* * * * *\", \"jitter\": \"5 min\"}\n config = schedule_parse.ConfigGenericSchedule(\n \"cron\",\n schedule[\"value\"],\n datetime.timedelta(minutes=5),\n )\n self.assert_validation(schedule, config)\n\n\nclass TestValidCronScheduler(TestCase):\n _suites = [\"integration\"]\n\n def validate(self, line):\n config = schedule_parse.ConfigGenericSchedule(\"cron\", line, None)\n context = config_utils.NullConfigContext\n return schedule_parse.valid_cron_scheduler(config, context)\n\n def test_valid_config(self):\n config = self.validate(\"5 0 L * *\")\n assert_equal(config.minutes, [5])\n assert_equal(config.months, None)\n assert_equal(config.monthdays, [\"LAST\"])\n\n def test_invalid_config(self):\n assert_raises(ConfigError, self.validate, \"* * *\")\n\n\nclass TestValidDailyScheduler(TestCase):\n def validate(self, config):\n context = config_utils.NullConfigContext\n config = schedule_parse.ConfigGenericSchedule(\"daily\", config, None)\n return schedule_parse.valid_daily_scheduler(config, context)\n\n def assert_parse(self, config, expected):\n config = self.validate(config)\n expected = schedule_parse.ConfigDailyScheduler(*expected, jitter=None)\n assert_equal(config, expected)\n\n def test_valid_daily_scheduler_start_time(self):\n expected = (\"14:32 \", 14, 32, 0, set())\n self.assert_parse(\"14:32\", expected)\n\n def test_valid_daily_scheduler_just_days(self):\n expected = (\"00:00:00 MWS\", 0, 0, 0, {1, 3, 6})\n self.assert_parse(\"00:00:00 MWS\", expected)\n\n def test_valid_daily_scheduler_time_and_day(self):\n expected = (\"17:02:44 SU\", 17, 2, 44, {0, 6})\n self.assert_parse(\"17:02:44 SU\", expected)\n\n def test_valid_daily_scheduler_invalid_start_time(self):\n assert_raises(ConfigError, self.validate, \"5 MWF\")\n assert_raises(ConfigError, self.validate, \"05:30:45:45 MWF\")\n assert_raises(ConfigError, self.validate, \"25:30:45 MWF\")\n\n def test_valid_daily_scheduler_invalid_days(self):\n assert_raises(ConfigError, self.validate, \"SUG\")\n assert_raises(ConfigError, self.validate, \"3\")\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/core/__init__.py", "content": "" }, { "path": "tests/core/action_test.py", "content": "import pytest\n\nfrom tron.config.schema import ConfigAction\nfrom tron.config.schema import ConfigConstraint\nfrom tron.config.schema import ConfigFieldSelectorSource\nfrom tron.config.schema import ConfigNodeAffinity\nfrom tron.config.schema import ConfigParameter\nfrom tron.config.schema import ConfigProjectedSAVolume\nfrom tron.config.schema import ConfigSecretSource\nfrom tron.config.schema import ConfigSecretVolume\nfrom tron.config.schema import ConfigSecretVolumeItem\nfrom tron.config.schema import ConfigTopologySpreadConstraints\nfrom tron.config.schema import ConfigVolume\nfrom tron.core.action import Action\nfrom tron.core.action import ActionCommandConfig\n\n\nclass TestAction:\n @pytest.mark.parametrize(\"disk\", [600.0, None])\n def test_from_config_full(self, disk):\n config = ConfigAction(\n name=\"ted\",\n command=\"do something\",\n node=\"first\",\n executor=\"ssh\",\n cpus=1,\n mem=100,\n disk=disk, # default: 1024.0\n constraints=[\n ConfigConstraint(\n attribute=\"pool\",\n operator=\"LIKE\",\n value=\"default\",\n ),\n ],\n docker_image=\"fake-docker.com:400/image\",\n docker_parameters=[\n ConfigParameter(\n key=\"test\",\n value=123,\n ),\n ],\n env={\"TESTING\": \"true\"},\n secret_env={\"TEST_SECRET\": ConfigSecretSource(secret_name=\"tron-secret-svc-sec--A\", key=\"sec_A\")},\n secret_volumes=[\n ConfigSecretVolume(\n secret_volume_name=\"secretvolumename\",\n secret_name=\"secret\",\n container_path=\"/b\",\n default_mode=\"0644\",\n items=[ConfigSecretVolumeItem(key=\"key\", path=\"path\", mode=\"0755\")],\n ),\n ],\n extra_volumes=[\n ConfigVolume(\n host_path=\"/tmp\",\n container_path=\"/nail/tmp\",\n mode=\"RO\",\n ),\n ],\n trigger_downstreams=True,\n triggered_by=[\"foo.bar\"],\n )\n new_action = Action.from_config(config)\n assert new_action.name == config.name\n assert new_action.node_pool is None\n assert new_action.executor == config.executor\n assert new_action.trigger_downstreams is True\n assert new_action.triggered_by == [\"foo.bar\"]\n\n command_config = new_action.command_config\n assert command_config.command == config.command\n assert command_config.cpus == config.cpus\n assert command_config.mem == config.mem\n assert command_config.disk == (600.0 if disk else 1024.0)\n assert command_config.constraints == {(\"pool\", \"LIKE\", \"default\")}\n assert command_config.docker_image == config.docker_image\n assert command_config.docker_parameters == {(\"test\", 123)}\n assert command_config.env == config.env\n assert command_config.secret_env == config.secret_env\n # cant do direct tuple equality, since this is not hashable\n assert command_config.secret_volumes == config.secret_volumes\n assert command_config.extra_volumes == {(\"/nail/tmp\", \"/tmp\", \"RO\")}\n\n def test_from_config_none_values(self):\n config = ConfigAction(\n name=\"ted\",\n command=\"do something\",\n node=\"first\",\n executor=\"ssh\",\n )\n new_action = Action.from_config(config)\n assert new_action.name == config.name\n assert new_action.executor == config.executor\n command_config = new_action.command_config\n assert command_config.command == config.command\n assert command_config.constraints == set()\n assert command_config.docker_image is None\n assert command_config.docker_parameters == set()\n assert command_config.env == {}\n assert command_config.secret_env == {}\n assert command_config.secret_volumes == []\n assert command_config.extra_volumes == set()\n\n @pytest.fixture\n def action_command_config_json(self):\n raw_json = \"\"\"\n {\n \"command\": \"echo 'Hello, World!'\",\n \"cpus\": 1.0,\n \"mem\": 512.0,\n \"disk\": 1024.0,\n \"cap_add\": [\"NET_ADMIN\"],\n \"cap_drop\": [\"MKNOD\"],\n \"constraints\": [\n {\n \"attribute\": \"pool\",\n \"operator\": \"LIKE\",\n \"value\": \"default\"\n }\n ],\n \"docker_image\": \"fake-docker.com:400/image\",\n \"docker_parameters\": [\n {\n \"key\": \"test\",\n \"value\": 123\n }\n ],\n \"env\": {\"TESTING\": \"true\"},\n \"secret_env\": {\n \"TEST_SECRET\": {\n \"secret_name\": \"tron-secret-svc-sec--A\",\n \"key\": \"sec_A\"\n }\n },\n \"secret_volumes\": [\n {\n \"secret_volume_name\": \"secretvolumename\",\n \"secret_name\": \"secret\",\n \"container_path\": \"/b\",\n \"default_mode\": \"0644\",\n \"items\": [\n {\n \"key\": \"key\",\n \"path\": \"path\",\n \"mode\": \"0755\"\n }\n ]\n }\n ],\n \"projected_sa_volumes\": [\n {\n \"container_path\": \"/var/run/secrets/whatever\",\n \"audience\": \"for.bar.com\",\n \"expiration_seconds\": 3600\n }\n ],\n \"extra_volumes\": [\n {\n \"container_path\": \"/tmp\",\n \"host_path\": \"/home/tmp\",\n \"mode\": \"RO\"\n }\n ],\n \"node_affinities\": [\n {\n \"key\": \"topology.kubernetes.io/zone\",\n \"operator\": \"In\",\n \"value\": [\"us-west-1a\", \"us-west-1c\"]\n }\n ],\n \"topology_spread_constraints\": [\n {\n \"topology_key\": \"zone\",\n \"max_skew\": 1,\n \"when_unsatisfiable\": \"DoNotSchedule\",\n \"label_selector\": {\n \"match_labels\": {\n \"app\": \"myapp\"\n }\n }\n }\n ],\n \"labels\": {\"app\": \"myapp\"},\n \"idempotent\": true,\n \"annotations\": {\"annotation_key\": \"annotation_value\"},\n \"service_account_name\": \"default\",\n \"ports\": [8080, 9090],\n \"field_selector_env\": {\n \"key\": {\n \"field_path\": \"value\"\n }\n },\n \"node_selectors\": {\"key\": \"node-A\"}\n }\n \"\"\"\n return raw_json\n\n def test_action_command_config_from_json(self, action_command_config_json):\n result = ActionCommandConfig.from_json(action_command_config_json)\n\n expected = {\n \"command\": \"echo 'Hello, World!'\",\n \"cpus\": 1.0,\n \"mem\": 512.0,\n \"disk\": 1024.0,\n \"cap_add\": [\"NET_ADMIN\"],\n \"cap_drop\": [\"MKNOD\"],\n \"constraints\": [ConfigConstraint(attribute=\"pool\", operator=\"LIKE\", value=\"default\")],\n \"docker_image\": \"fake-docker.com:400/image\",\n \"docker_parameters\": [ConfigParameter(key=\"test\", value=123)],\n \"env\": {\"TESTING\": \"true\"},\n \"secret_env\": {\"TEST_SECRET\": ConfigSecretSource(secret_name=\"tron-secret-svc-sec--A\", key=\"sec_A\")},\n \"secret_volumes\": [\n ConfigSecretVolume(\n secret_volume_name=\"secretvolumename\",\n secret_name=\"secret\",\n container_path=\"/b\",\n default_mode=\"0644\",\n items=[{\"key\": \"key\", \"path\": \"path\", \"mode\": \"0755\"}],\n )\n ],\n \"projected_sa_volumes\": [\n ConfigProjectedSAVolume(\n container_path=\"/var/run/secrets/whatever\",\n audience=\"for.bar.com\",\n expiration_seconds=3600,\n )\n ],\n \"extra_volumes\": [ConfigVolume(container_path=\"/tmp\", host_path=\"/home/tmp\", mode=\"RO\")],\n \"node_affinities\": [\n ConfigNodeAffinity(key=\"topology.kubernetes.io/zone\", operator=\"In\", value=[\"us-west-1a\", \"us-west-1c\"])\n ],\n \"topology_spread_constraints\": [\n ConfigTopologySpreadConstraints(\n topology_key=\"zone\",\n max_skew=1,\n when_unsatisfiable=\"DoNotSchedule\",\n label_selector={\"match_labels\": {\"app\": \"myapp\"}},\n )\n ],\n \"labels\": {\"app\": \"myapp\"},\n \"annotations\": {\"annotation_key\": \"annotation_value\"},\n \"service_account_name\": \"default\",\n \"ports\": [8080, 9090],\n \"idempotent\": True,\n \"node_selectors\": {\"key\": \"node-A\"},\n \"field_selector_env\": {\"key\": ConfigFieldSelectorSource(field_path=\"value\")},\n }\n\n assert result == expected\n" }, { "path": "tests/core/actiongraph_test.py", "content": "from unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_raises\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tron.core import actiongraph\n\n\nclass TestActionGraph(TestCase):\n @setup\n def setup_graph(self):\n self.action_names = [\n \"base_one\",\n \"base_two\",\n \"dep_one\",\n \"dep_one_one\",\n \"dep_multi\",\n ]\n self.action_map = {}\n for name in self.action_names:\n self.action_map[name] = mock.MagicMock()\n self.action_map[name].name = name\n\n self.required_actions = {\n \"base_one\": set(),\n \"base_two\": set(),\n \"dep_multi\": {\"dep_one_one\", \"base_two\"},\n \"dep_one_one\": {\"dep_one\"},\n \"dep_one\": {\"base_one\"},\n }\n self.required_triggers = {\n \"base_one\": {\"MASTER.otherjob.first\"},\n \"base_two\": set(),\n \"dep_multi\": set(),\n \"dep_one_one\": set(),\n \"dep_one\": set(),\n }\n\n self.action_graph = actiongraph.ActionGraph(self.action_map, self.required_actions, self.required_triggers)\n\n def test_get_dependencies(self):\n assert self.action_graph.get_dependencies(\"not_in_job\") == []\n assert self.action_graph.get_dependencies(\"base_one\") == []\n assert self.action_graph.get_dependencies(\"base_one\", include_triggers=True)[0].name == \"MASTER.otherjob.first\"\n assert sorted(d.name for d in self.action_graph.get_dependencies(\"dep_multi\")) == sorted(\n [\n \"dep_one_one\",\n \"base_two\",\n ]\n )\n\n def test_names(self):\n assert sorted(self.action_graph.names()) == sorted(self.action_names)\n assert sorted(self.action_graph.names(include_triggers=True)) == sorted(\n self.action_names + [\"MASTER.otherjob.first\"],\n )\n\n def test__getitem__(self):\n assert_equal(\n self.action_graph[\"base_one\"],\n self.action_map[\"base_one\"],\n )\n\n def test__getitem__miss(self):\n assert_raises(KeyError, lambda: self.action_graph[\"unknown\"])\n\n def test__eq__(self):\n other_graph = mock.MagicMock(\n action_map=self.action_map,\n required_actions=self.required_actions,\n required_triggers=self.required_triggers,\n )\n assert_equal(self.action_graph, other_graph)\n\n other_graph.required_actions = None\n assert not self.action_graph == other_graph\n\n def test__ne__(self):\n other_graph = mock.MagicMock()\n assert self.action_graph != other_graph\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/core/actionrun_test.py", "content": "import datetime\nimport shutil\nimport tempfile\nfrom unittest import mock\nfrom unittest.mock import MagicMock\n\nimport pytest\n\nfrom tests.assertions import assert_length\nfrom tests.testingutils import autospec_method\nfrom tron import actioncommand\nfrom tron import node\nfrom tron.actioncommand import SubprocessActionRunnerFactory\nfrom tron.config.schema import ConfigConstraint\nfrom tron.config.schema import ConfigParameter\nfrom tron.config.schema import ConfigVolume\nfrom tron.config.schema import ExecutorTypes\nfrom tron.core import actiongraph\nfrom tron.core import jobrun\nfrom tron.core.action import ActionCommandConfig\nfrom tron.core.actionrun import ActionCommand\nfrom tron.core.actionrun import ActionRun\nfrom tron.core.actionrun import ActionRunAttempt\nfrom tron.core.actionrun import ActionRunCollection\nfrom tron.core.actionrun import ActionRunFactory\nfrom tron.core.actionrun import eager_all\nfrom tron.core.actionrun import INITIAL_RECOVER_DELAY\nfrom tron.core.actionrun import KubernetesActionRun\nfrom tron.core.actionrun import MAX_RECOVER_TRIES\nfrom tron.core.actionrun import MesosActionRun\nfrom tron.core.actionrun import min_filter\nfrom tron.core.actionrun import SSHActionRun\nfrom tron.serialize import filehandler\n\n\n@pytest.fixture\ndef output_path():\n output_path = filehandler.OutputPath(tempfile.mkdtemp())\n yield output_path\n shutil.rmtree(output_path.base, ignore_errors=True)\n\n\n@pytest.fixture\ndef mock_current_time():\n with mock.patch(\n \"tron.core.actionrun.timeutils.current_time\",\n autospec=True,\n ) as mock_current_time:\n yield mock_current_time\n\n\nclass TestMinFilter:\n def test_min_filter(self):\n seq = [None, 2, None, 7, None, 9, 10, 12, 1]\n assert min_filter(seq) == 1\n\n\nclass TestEagerAll:\n def test_all_true(self):\n assert eager_all(range(1, 5))\n\n def test_all_false(self):\n assert not eager_all(0 for _ in range(7))\n\n def test_full_iteration(self):\n seq = iter([1, 0, 3, 0, 5])\n assert not eager_all(seq)\n with pytest.raises(StopIteration):\n next(seq)\n\n\nclass TestActionRunFactory:\n @pytest.fixture(autouse=True)\n def setup_action_runs(self):\n self.run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)\n a1 = MagicMock()\n a1.name = \"act1\"\n a1.command_config = ActionCommandConfig(command=\"do action1\")\n a2 = MagicMock()\n a2.name = \"act2\"\n actions = [a1, a2]\n self.action_graph = actiongraph.ActionGraph(\n {a.name: a for a in actions},\n {\"act1\": set(), \"act2\": set()},\n {\"act1\": set(), \"act2\": set()},\n )\n\n mock_node = mock.create_autospec(node.Node)\n self.job_run = jobrun.JobRun(\n \"jobname\",\n 7,\n self.run_time,\n mock_node,\n action_graph=self.action_graph,\n )\n\n self.action_runner = mock.create_autospec(\n actioncommand.SubprocessActionRunnerFactory,\n )\n\n @pytest.fixture\n def state_data(self):\n command_config = self.action_graph.action_map[\"act1\"].command_config.state_data\n # State data with command config and retries.\n yield {\n \"job_run_id\": \"job_run_id\",\n \"action_name\": \"act1\",\n \"state\": \"succeeded\",\n \"run_time\": \"the_run_time\",\n \"start_time\": None,\n \"end_time\": None,\n \"attempts\": [dict(command_config=command_config, start_time=\"start\")],\n \"node_name\": \"anode\",\n }\n\n def test_build_action_run_collection(self):\n collection = ActionRunFactory.build_action_run_collection(\n self.job_run,\n self.action_runner,\n )\n assert collection.action_graph == self.action_graph\n assert \"act1\" in collection.run_map\n assert \"act2\" in collection.run_map\n assert len(collection.run_map) == 2\n assert collection.run_map[\"act1\"].action_name == \"act1\"\n\n def test_action_run_collection_from_state(self, state_data):\n state_data = [state_data]\n cleanup_command_config = dict(command=\"do action1\")\n cleanup_action_state_data = {\n \"job_run_id\": \"job_run_id\",\n \"action_name\": \"cleanup\",\n \"state\": \"succeeded\",\n \"run_time\": self.run_time,\n \"start_time\": None,\n \"end_time\": None,\n \"attempts\": [\n dict(\n command_config=cleanup_command_config,\n rendered_command=\"do action1\",\n start_time=\"start\",\n end_time=\"end\",\n exit_status=0,\n ),\n ],\n \"node_name\": \"anode\",\n \"action_runner\": {\n \"status_path\": \"/tmp/foo\",\n \"exec_path\": \"/bin/foo\",\n },\n }\n collection = ActionRunFactory.action_run_collection_from_state(\n self.job_run,\n state_data,\n cleanup_action_state_data,\n )\n\n assert collection.action_graph == self.action_graph\n assert_length(collection.run_map, 2)\n assert collection.run_map[\"act1\"].action_name == \"act1\"\n assert collection.run_map[\"cleanup\"].action_name == \"cleanup\"\n\n def test_build_run_for_action(self):\n expected_command = \"doit\"\n action = MagicMock(\n node_pool=None,\n is_cleanup=False,\n command_config=ActionCommandConfig(command=expected_command),\n )\n action.name = \"theaction\"\n action_run = ActionRunFactory.build_run_for_action(\n self.job_run,\n action,\n self.action_runner,\n )\n\n assert action_run.job_run_id == self.job_run.id\n assert action_run.node == self.job_run.node\n assert action_run.action_name == action.name\n assert not action_run.is_cleanup\n assert action_run.command == expected_command\n\n def test_build_run_for_action_with_node(self):\n expected_command = \"doit\"\n action = MagicMock(\n node_pool=None,\n is_cleanup=True,\n command_config=ActionCommandConfig(command=expected_command),\n )\n action.node_pool = mock.create_autospec(node.NodePool)\n action_run = ActionRunFactory.build_run_for_action(\n self.job_run,\n action,\n self.action_runner,\n )\n\n assert action_run.job_run_id == self.job_run.id\n assert action_run.node == action.node_pool.next()\n assert action_run.is_cleanup\n assert action_run.action_name == action.name\n assert action_run.command == expected_command\n\n def test_build_run_for_ssh_action(self):\n action = MagicMock(\n name=\"theaction\",\n command=\"doit\",\n executor=ExecutorTypes.ssh.value,\n )\n action_run = ActionRunFactory.build_run_for_action(\n self.job_run,\n action,\n self.action_runner,\n )\n assert action_run.__class__ == SSHActionRun\n\n def test_build_run_for_mesos_action(self):\n command_config = MagicMock(\n cpus=10,\n mem=500,\n disk=600,\n constraints=[[\"pool\", \"LIKE\", \"default\"]],\n docker_image=\"fake-docker.com:400/image\",\n docker_parameters=[\n {\n \"key\": \"test\",\n \"value\": 123,\n }\n ],\n env={\"TESTING\": \"true\"},\n extra_volumes=[\n {\n \"path\": \"/tmp\",\n }\n ],\n )\n action = MagicMock(\n name=\"theaction\",\n command=\"doit\",\n executor=ExecutorTypes.mesos.value,\n command_config=command_config,\n )\n action_run = ActionRunFactory.build_run_for_action(\n self.job_run,\n action,\n self.action_runner,\n )\n assert action_run.__class__ == MesosActionRun\n assert action_run.command_config.cpus == command_config.cpus\n assert action_run.command_config.mem == command_config.mem\n assert action_run.command_config.disk == command_config.disk\n assert action_run.command_config.constraints == command_config.constraints\n assert action_run.command_config.docker_image == command_config.docker_image\n assert action_run.command_config.docker_parameters == command_config.docker_parameters\n assert action_run.command_config.env == command_config.env\n assert action_run.command_config.extra_volumes == command_config.extra_volumes\n\n def test_action_run_from_state_ssh(self, state_data):\n action_run = ActionRunFactory.action_run_from_state(\n self.job_run,\n state_data,\n )\n\n assert action_run.job_run_id == state_data[\"job_run_id\"]\n assert not action_run.is_cleanup\n assert action_run.__class__ == SSHActionRun\n\n def test_action_run_from_state_mesos(self, state_data):\n state_data[\"executor\"] = ExecutorTypes.mesos.value\n action_run = ActionRunFactory.action_run_from_state(\n self.job_run,\n state_data,\n )\n\n assert action_run.job_run_id == state_data[\"job_run_id\"]\n action_name = state_data[\"action_name\"]\n assert action_run.command_config == self.action_graph.action_map[action_name].command_config\n\n assert not action_run.is_cleanup\n assert action_run.__class__ == MesosActionRun\n\n def test_action_run_from_state_kubernetes(self, state_data):\n state_data[\"executor\"] = ExecutorTypes.kubernetes.value\n action_run = ActionRunFactory.action_run_from_state(\n self.job_run,\n state_data,\n )\n\n assert action_run.job_run_id == state_data[\"job_run_id\"]\n action_name = state_data[\"action_name\"]\n assert action_run.command_config == self.action_graph.action_map[action_name].command_config\n\n assert not action_run.is_cleanup\n assert action_run.__class__ == KubernetesActionRun\n\n def test_action_run_from_state_spark(self, state_data):\n state_data[\"executor\"] = ExecutorTypes.spark.value\n action_run = ActionRunFactory.action_run_from_state(\n self.job_run,\n state_data,\n )\n\n assert action_run.job_run_id == state_data[\"job_run_id\"]\n action_name = state_data[\"action_name\"]\n assert action_run.command_config == self.action_graph.action_map[action_name].command_config\n\n assert not action_run.is_cleanup\n assert action_run.__class__ == KubernetesActionRun\n\n\nclass TestActionRun:\n @pytest.fixture(autouse=True)\n def setup_action_run(self, output_path):\n self.action_runner = actioncommand.NoActionRunnerFactory()\n self.command = \"do command {actionname}\"\n self.rendered_command = \"do command action_name\"\n self.action_run = ActionRun(\n job_run_id=\"ns.id.0\",\n name=\"action_name\",\n node=mock.create_autospec(node.Node),\n command_config=ActionCommandConfig(command=self.command),\n output_path=output_path,\n action_runner=self.action_runner,\n )\n # These should be implemented in subclasses, we don't care here\n self.action_run.submit_command = mock.Mock()\n self.action_run.stop = mock.Mock()\n self.action_run.kill = mock.Mock()\n\n def test_init_state(self):\n assert self.action_run.state == ActionRun.SCHEDULED\n\n def test_ready_state(self):\n self.action_run.ready()\n assert self.action_run.state == ActionRun.WAITING\n\n def test_start(self):\n self.action_run.machine.transition(\"ready\")\n assert self.action_run.start()\n assert self.action_run.submit_command.call_count == 1\n assert self.action_run.is_starting\n assert self.action_run.start_time\n\n def test_start_bad_state(self):\n self.action_run.fail()\n assert not self.action_run.start()\n\n @mock.patch(\"tron.core.actionrun.log\", autospec=True)\n def test_start_invalid_command(self, _log):\n self.action_run.original_command = \"{notfound}\"\n self.action_run.machine.transition(\"ready\")\n assert not self.action_run.start()\n assert self.action_run.is_failed\n assert self.action_run.exit_status == -1\n\n def test_success(self):\n assert self.action_run.ready()\n self.action_run.machine.transition(\"start\")\n self.action_run.machine.transition(\"started\")\n\n assert self.action_run.is_running\n assert self.action_run.success()\n assert not self.action_run.is_running\n assert self.action_run.is_done\n assert self.action_run.end_time\n assert self.action_run.exit_status == 0\n\n def test_success_emits_not(self):\n self.action_run.machine.transition(\"start\")\n self.action_run.machine.transition(\"started\")\n self.action_run.trigger_downstreams = None\n self.action_run.emit_triggers = mock.Mock()\n assert self.action_run.success()\n assert self.action_run.emit_triggers.call_count == 0\n\n def test_sucess_emits_not_invalid_transition(self):\n self.action_run.trigger_downstreams = True\n self.action_run.machine.check = mock.Mock(return_value=False)\n self.action_run.emit_triggers = mock.Mock()\n\n assert not self.action_run.success()\n assert self.action_run.emit_triggers.call_count == 0\n\n def test_success_emits_on_true(self):\n self.action_run.machine.transition(\"start\")\n self.action_run.machine.transition(\"started\")\n self.action_run.trigger_downstreams = True\n self.action_run.emit_triggers = mock.Mock()\n assert self.action_run.success()\n assert self.action_run.emit_triggers.call_count == 1\n\n def test_success_emits_on_dict(self):\n self.action_run.machine.transition(\"start\")\n self.action_run.machine.transition(\"started\")\n self.action_run.trigger_downstreams = dict(foo=\"bar\")\n self.action_run.emit_triggers = mock.Mock()\n assert self.action_run.success()\n assert self.action_run.emit_triggers.call_count == 1\n\n @mock.patch(\"tron.core.actionrun.EventBus\", autospec=True)\n def test_emit_triggers(self, eventbus):\n self.action_run.context = {\"shortdate\": \"foo\"}\n\n self.action_run.trigger_downstreams = True\n self.action_run.emit_triggers()\n\n self.action_run.trigger_downstreams = dict(foo=\"bar\")\n self.action_run.emit_triggers()\n\n assert eventbus.publish.mock_calls == [\n mock.call(\"ns.id.action_name.shortdate.foo\"),\n mock.call(\"ns.id.action_name.foo.bar\"),\n ]\n\n def test_failure(self):\n self.action_run._exit_unsuccessful(1)\n assert not self.action_run.is_running\n assert self.action_run.is_done\n assert self.action_run.end_time\n assert self.action_run.exit_status == 1\n\n def test_failure_bad_state(self):\n self.action_run.fail(444)\n assert not self.action_run.fail(123)\n assert self.action_run.exit_status == 444\n\n def test_skip(self):\n assert not self.action_run.is_running\n self.action_run.ready()\n assert self.action_run.start()\n\n assert self.action_run.fail(-1)\n assert self.action_run.skip()\n assert self.action_run.is_skipped\n\n def test_skip_bad_state(self):\n assert not self.action_run.skip()\n\n def test_render_command(self):\n self.action_run.context = {\"stars\": \"bright\"}\n bare_command = \"{stars}\"\n assert self.action_run.render_command(bare_command) == \"bright\"\n\n def test_command_not_yet_rendered(self):\n assert self.action_run.command == self.action_run.command_config.command\n\n def test_command_already_rendered(self):\n last_attempt = self.action_run.create_attempt()\n assert self.action_run.command == last_attempt.rendered_command\n\n @mock.patch(\"tron.core.actionrun.log\", autospec=True)\n def test_command_failed_render(self, _log):\n bare_command = \"{this_is_missing}\"\n assert self.action_run.render_command(bare_command) == ActionRun.FAILED_RENDER\n\n def test_is_complete(self):\n self.action_run.machine.state = ActionRun.SUCCEEDED\n assert self.action_run.is_complete\n self.action_run.machine.state = ActionRun.SKIPPED\n assert self.action_run.is_complete\n self.action_run.machine.state = ActionRun.RUNNING\n assert not self.action_run.is_complete\n\n def test_is_broken(self):\n self.action_run.machine.state = ActionRun.UNKNOWN\n assert self.action_run.is_broken\n self.action_run.machine.state = ActionRun.FAILED\n assert self.action_run.is_broken\n self.action_run.machine.state = ActionRun.WAITING\n assert not self.action_run.is_broken\n\n def test__getattr__(self):\n assert not self.action_run.is_succeeded\n assert not self.action_run.is_failed\n assert not self.action_run.is_queued\n assert self.action_run.is_scheduled\n assert self.action_run.cancel()\n assert self.action_run.is_cancelled\n\n def test__getattr__missing_attribute(self):\n with pytest.raises(AttributeError):\n self.action_run.__getattr__(\"is_not_a_real_state\")\n\n def test_auto_retry(self, mock_current_time):\n # One timestamp for start and end of each attempt, plus final end time\n mock_current_time.side_effect = [1, 2, 3, 4, 5, 6, 7]\n self.action_run.retries_remaining = 2\n self.action_run.create_attempt()\n self.action_run.machine.transition(\"start\")\n\n assert self.action_run._exit_unsuccessful(-1)\n assert self.action_run.is_starting\n assert self.action_run.retries_remaining == 1\n\n assert self.action_run._exit_unsuccessful(-1)\n assert self.action_run.retries_remaining == 0\n assert not self.action_run.is_failed\n\n assert self.action_run._exit_unsuccessful(-2)\n assert self.action_run.retries_remaining == 0\n assert self.action_run.is_failed\n\n assert self.action_run.exit_statuses == [-1, -1, -2]\n assert len(self.action_run.attempts) == 3\n for i, attempt in enumerate(self.action_run.attempts):\n assert attempt.start_time == i * 2 + 1\n assert attempt.end_time == (i + 1) * 2\n\n def test_auto_retry_command_config_change(self, mock_current_time):\n self.action_run.retries_remaining = 1\n self.action_run.create_attempt()\n self.action_run.machine.transition(\"start\")\n\n # If the command_config gets reconfigured later, auto retry\n # still uses the original command by default.\n self.action_run.command_config = ActionCommandConfig(command=\"new\")\n\n assert self.action_run._exit_unsuccessful(-1)\n assert self.action_run._exit_unsuccessful(-1)\n assert len(self.action_run.attempts) == 2\n\n for i, attempt in enumerate(self.action_run.attempts):\n assert attempt.rendered_command == self.rendered_command\n\n def test_no_auto_retry_on_fail_not_running(self):\n self.action_run.retries_remaining = 2\n\n self.action_run.fail()\n assert self.action_run.retries_remaining == -1\n assert self.action_run.is_failed\n\n assert self.action_run.exit_statuses == []\n assert self.action_run.exit_status is None\n\n def test_no_auto_retry_on_fail_running(self):\n self.action_run.retries_remaining = 2\n self.action_run.create_attempt()\n self.action_run.machine.transition(\"start\")\n\n self.action_run.fail()\n assert self.action_run.retries_remaining == -1\n assert self.action_run.is_failed\n\n assert self.action_run.exit_statuses == [None]\n assert self.action_run.exit_status is None\n\n def test_auto_retry_already_done(self):\n # If someone transitions the action before it\n # is done to success/fail, the action\n # should not automatically retry when the command\n # completes.\n self.action_run.retries_remaining = 2\n self.action_run.create_attempt()\n self.action_run.machine.transition(\"start\")\n self.action_run.machine.transition(\"started\")\n\n # Action gets manually transitioned to success with tronctl\n self.action_run.machine.transition(\"success\")\n assert self.action_run.is_succeeded\n\n # Command later fails\n # Does not start a command\n assert not self.action_run._exit_unsuccessful(-1)\n # Still succeeded, not starting\n assert self.action_run.is_succeeded\n\n def test_manual_retry(self, mock_current_time):\n mock_current_time.side_effect = [1, 2, 3, 4]\n self.action_run.retries_remaining = None\n failed_attempt = self.action_run.create_attempt()\n self.action_run.machine.transition(\"start\")\n self.action_run.fail(-1)\n assert failed_attempt.end_time == 2\n assert failed_attempt.exit_status == -1\n\n self.action_run.retry()\n assert self.action_run.is_starting\n assert self.action_run.exit_statuses == [-1]\n assert self.action_run.retries_remaining == 0\n # Last attempt should be unchanged\n assert failed_attempt.end_time == 2\n assert failed_attempt.exit_status == -1\n\n def test_manual_retry_use_new_command(self, mock_current_time):\n mock_current_time.side_effect = [1, 2, 3, 4]\n self.action_run.retries_remaining = None\n self.action_run.create_attempt()\n self.action_run.machine.transition(\"start\")\n self.action_run.fail(-1)\n\n # Change the command config\n self.action_run.command_config = ActionCommandConfig(command=\"new\")\n self.action_run.retry(original_command=False)\n assert self.action_run.is_starting\n assert self.action_run.last_attempt.rendered_command == \"new\"\n\n @mock.patch(\"twisted.internet.reactor.callLater\", autospec=True)\n def test_retries_delay(self, callLater):\n self.action_run.retries_delay = datetime.timedelta()\n self.action_run.retries_remaining = 2\n self.action_run.machine.transition(\"start\")\n callLater.return_value = \"delayed call\"\n assert self.action_run._exit_unsuccessful(-1)\n assert self.action_run.in_delay == \"delayed call\"\n\n\nclass TestActionRunFactoryTriggerTimeout:\n def test_trigger_timeout_default(self):\n today = datetime.datetime.today()\n day = datetime.timedelta(days=1)\n tomorrow = today + day\n action_run = ActionRunFactory.build_run_for_action(\n mock.Mock(run_time=today),\n mock.Mock(trigger_timeout=None),\n mock.Mock(),\n )\n assert action_run.trigger_timeout_timestamp == tomorrow.timestamp()\n\n def test_trigger_timeout_custom(self):\n today = datetime.datetime.today()\n hour = datetime.timedelta(hours=1)\n target = today + hour\n action_run = ActionRunFactory.build_run_for_action(\n mock.Mock(run_time=today),\n mock.Mock(trigger_timeout=hour),\n mock.Mock(),\n )\n assert action_run.trigger_timeout_timestamp == target.timestamp()\n\n\nclass TestActionRunTriggerTimeout:\n @pytest.fixture(autouse=True)\n def setup_teardown(self):\n self.command = mock.Mock()\n self.rendered_command = \"do command action_name\"\n self.action_run = ActionRun(\n job_run_id=\"ns.id.0\",\n name=\"action_name\",\n command_config=ActionCommandConfig(command=self.command),\n triggered_by=[\"hello\"],\n node=mock.Mock(),\n output_path=mock.Mock(),\n action_runner=mock.Mock(),\n trigger_timeout_timestamp=mock.Mock(),\n )\n self.action_run.submit_command = mock.Mock()\n self.action_run.stop = mock.Mock()\n self.action_run.kill = mock.Mock()\n\n def test_cleanup_clears_trigger_timeout(self):\n self.action_run.clear_trigger_timeout = MagicMock()\n self.action_run.cleanup()\n self.action_run.clear_trigger_timeout.assert_called_with()\n\n def test_clear_trigger_timeout(self):\n timeout_call = MagicMock()\n self.action_run.trigger_timeout_call = timeout_call\n self.action_run.clear_trigger_timeout()\n assert self.action_run.trigger_timeout_call is None\n timeout_call.cancel.assert_called_with()\n\n @mock.patch(\"tron.core.actionrun.EventBus\", autospec=True)\n @mock.patch(\"tron.core.actionrun.reactor\", autospec=True)\n def test_setup_subscriptions_no_triggers(self, reactor, eventbus):\n self.action_run.triggered_by = []\n self.action_run.setup_subscriptions()\n assert not reactor.callLater.called\n assert not eventbus.subscribe.called\n\n @mock.patch(\"tron.core.actionrun.EventBus\", autospec=True)\n @mock.patch(\"tron.core.actionrun.reactor\", autospec=True)\n def test_setup_subscriptions_no_remaining(self, reactor, eventbus):\n self.action_run.triggered_by = [\"hello\"]\n self.action_run.trigger_timeout_timestamp = None\n eventbus.has_event.return_value = True\n self.action_run.setup_subscriptions()\n assert not reactor.callLater.called\n assert not eventbus.subscribe.called\n assert eventbus.has_event.call_args_list == [mock.call(\"hello\")]\n\n @mock.patch(\"tron.core.actionrun.reactor\", autospec=True)\n def test_setup_subscriptions_timeout_in_future(self, reactor, mock_current_time):\n now = datetime.datetime.now()\n mock_current_time.return_value = now\n self.action_run.trigger_timeout_timestamp = now.timestamp() + 10\n self.action_run.setup_subscriptions()\n reactor.callLater.assert_called_once_with(\n 10.0,\n self.action_run.trigger_timeout_reached,\n )\n\n @mock.patch(\"tron.core.actionrun.reactor\", autospec=True)\n def test_setup_subscriptions_timeout_in_past(self, reactor, mock_current_time):\n now = datetime.datetime.now()\n mock_current_time.return_value = now\n self.action_run.trigger_timeout_timestamp = now.timestamp() - 10\n self.action_run.setup_subscriptions()\n reactor.callLater.assert_called_once_with(\n 1,\n self.action_run.trigger_timeout_reached,\n )\n\n @mock.patch(\"tron.core.actionrun.EventBus\", autospec=True)\n def test_trigger_timeout_reached_no_remaining_notifies(self, eventbus):\n self.action_run.notify = MagicMock()\n self.action_run.triggered_by = [\"hello\"]\n eventbus.has_event.return_value = True\n self.action_run.trigger_timeout_reached()\n assert self.action_run.notify.called\n\n @mock.patch(\"tron.core.actionrun.EventBus\", autospec=True)\n def test_trigger_timeout_reached_with_remaining_fails(self, eventbus):\n self.action_run.fail = MagicMock()\n self.action_run.triggered_by = [\"hello\"]\n eventbus.has_event.return_value = False\n self.action_run.trigger_timeout_reached()\n assert self.action_run.fail.called\n\n def test_done_clears_trigger_timeout_call(self):\n self.action_run.machine.check = mock.Mock(return_value=True)\n self.action_run.transition_and_notify = MagicMock()\n self.action_run.triggered_by = []\n self.action_run.clear_trigger_timeout = MagicMock()\n self.action_run._done(ActionRun.SUCCEEDED)\n assert self.action_run.clear_trigger_timeout.called\n\n def test_trigger_notify_clears_trigger_timeout(self):\n self.action_run.notify = MagicMock()\n self.action_run.triggered_by = []\n self.action_run.clear_trigger_timeout = MagicMock()\n self.action_run.trigger_notify()\n assert self.action_run.clear_trigger_timeout.called\n\n\nclass TestSSHActionRun:\n @pytest.fixture(autouse=True)\n def setup_action_run(self, output_path):\n self.action_runner = mock.create_autospec(\n actioncommand.NoActionRunnerFactory,\n )\n self.command = \"do command {actionname}\"\n self.action_run = SSHActionRun(\n job_run_id=\"job_name.5\",\n name=\"action_name\",\n command_config=ActionCommandConfig(command=self.command),\n node=mock.create_autospec(node.Node),\n output_path=output_path,\n action_runner=self.action_runner,\n )\n\n def test_start_node_error(self):\n def raise_error(c):\n raise node.Error(\"The error\")\n\n self.action_run.node = mock.MagicMock()\n self.action_run.node.submit_command.side_effect = raise_error\n self.action_run.machine.transition(\"ready\")\n assert not self.action_run.start()\n assert self.action_run.exit_status == -2\n assert self.action_run.is_failed\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n def test_build_action_command(self, mock_filehandler):\n self.action_run.watch = mock.MagicMock()\n attempt = self.action_run.create_attempt()\n serializer = mock_filehandler.OutputStreamSerializer.return_value\n action_command = self.action_run.build_action_command(attempt)\n assert action_command == self.action_run.action_command\n assert action_command == self.action_runner.create.return_value\n self.action_runner.create.assert_called_with(\n self.action_run.id,\n attempt.rendered_command,\n serializer,\n )\n mock_filehandler.OutputStreamSerializer.assert_called_with(\n self.action_run.output_path,\n )\n self.action_run.watch.assert_called_with(action_command)\n\n def test_handler_running(self):\n attempt = self.action_run.create_attempt()\n self.action_run.build_action_command(attempt)\n self.action_run.machine.transition(\"start\")\n assert self.action_run.handler(\n self.action_run.action_command,\n ActionCommand.RUNNING,\n )\n assert self.action_run.is_running\n\n def test_handler_failstart(self):\n attempt = self.action_run.create_attempt()\n self.action_run.build_action_command(attempt)\n assert self.action_run.handler(\n self.action_run.action_command,\n ActionCommand.FAILSTART,\n )\n assert self.action_run.is_failed\n\n def test_handler_exiting_fail(self):\n attempt = self.action_run.create_attempt()\n self.action_run.build_action_command(attempt)\n self.action_run.action_command.exit_status = -1\n self.action_run.machine.transition(\"start\")\n assert self.action_run.handler(\n self.action_run.action_command,\n ActionCommand.EXITING,\n )\n assert self.action_run.is_failed\n assert self.action_run.exit_status == -1\n\n def test_handler_exiting_success(self):\n attempt = self.action_run.create_attempt()\n self.action_run.build_action_command(attempt)\n self.action_run.action_command.exit_status = 0\n self.action_run.machine.transition(\"start\")\n self.action_run.machine.transition(\"started\")\n assert self.action_run.handler(\n self.action_run.action_command,\n ActionCommand.EXITING,\n )\n assert self.action_run.is_succeeded\n assert self.action_run.exit_status == 0\n\n def test_handler_exiting_failunknown(self):\n self.action_run.action_command = mock.create_autospec(\n actioncommand.ActionCommand,\n exit_status=None,\n )\n self.action_run.machine.transition(\"start\")\n self.action_run.machine.transition(\"started\")\n assert self.action_run.handler(\n self.action_run.action_command,\n ActionCommand.EXITING,\n )\n assert self.action_run.is_unknown\n assert self.action_run.exit_status is None\n assert self.action_run.end_time is not None\n\n def test_handler_unhandled(self):\n attempt = self.action_run.create_attempt()\n self.action_run.build_action_command(attempt)\n assert (\n self.action_run.handler(\n self.action_run.action_command,\n ActionCommand.PENDING,\n )\n is None\n )\n assert self.action_run.is_scheduled\n\n def test_recover_no_action_runner(self):\n # Default setup has no action runner\n assert not self.action_run.recover()\n\n\nclass TestSSHActionRunRecover:\n @pytest.fixture(autouse=True)\n def setup_action_run(self, output_path):\n self.action_runner = SubprocessActionRunnerFactory(\n status_path=\"/tmp/foo\",\n exec_path=\"/bin/foo\",\n )\n self.command = \"do command {actionname}\"\n self.action_run = SSHActionRun(\n job_run_id=\"job_name.5\",\n name=\"action_name\",\n command_config=ActionCommandConfig(self.command),\n node=mock.create_autospec(node.Node),\n output_path=output_path,\n action_runner=self.action_runner,\n )\n\n def test_recover_incorrect_state(self):\n # Should return falsy if not UNKNOWN.\n self.action_run.machine.state = ActionRun.FAILED\n assert not self.action_run.recover()\n\n def test_recover_action_runner(self):\n self.action_run.end_time = 1000\n self.action_run.exit_status = 0\n self.action_run.machine.state = ActionRun.UNKNOWN\n last_attempt = self.action_run.create_attempt()\n last_attempt.end_time = 1000\n last_attempt.exit_status = 0\n assert self.action_run.recover()\n assert self.action_run.machine.state == ActionRun.RUNNING\n assert self.action_run.end_time is None\n assert self.action_run.exit_status is None\n assert last_attempt.end_time is None\n assert last_attempt.exit_status is None\n self.action_run.node.submit_command.assert_called_once()\n\n # Check recovery command\n submit_args = self.action_run.node.submit_command.call_args[0]\n assert len(submit_args) == 1\n recovery_command = submit_args[0]\n assert recovery_command.command == \"/bin/foo/recover_batch.py /tmp/foo/job_name.5.action_name/status\"\n assert recovery_command.start_time is not None # already started\n\n @mock.patch(\"tron.core.actionrun.reactor\", autospec=True)\n def test_handler_exiting_failunknown(self, mock_reactor):\n self.action_run.action_command = mock.create_autospec(\n actioncommand.ActionCommand,\n exit_status=None,\n )\n self.action_run.machine.transition(\"start\")\n self.action_run.machine.transition(\"started\")\n delay_deferred = self.action_run.handler(\n self.action_run.action_command,\n ActionCommand.EXITING,\n )\n assert delay_deferred == mock_reactor.callLater.return_value\n assert self.action_run.is_running\n assert self.action_run.exit_status is None\n assert self.action_run.end_time is None\n\n call_args = mock_reactor.callLater.call_args[0]\n assert call_args[0] == INITIAL_RECOVER_DELAY\n assert call_args[1] == self.action_run.submit_recovery_command\n\n # Check recovery run\n recovery_run = call_args[2]\n assert \"recovery\" in recovery_run.name\n assert isinstance(recovery_run, SSHActionRun)\n # Recovery run should not be recovering itself, parent run handles its unknown status\n assert recovery_run.recover() is None\n\n # Check command\n recovery_command = call_args[3]\n assert recovery_command.command == \"/bin/foo/recover_batch.py /tmp/foo/job_name.5.action_name/status\"\n assert recovery_command.start_time is not None # already started\n\n @mock.patch(\"tron.core.actionrun.SSHActionRun.do_recover\", autospec=True)\n @mock.patch(\"tron.core.actionrun.reactor\", autospec=True)\n def test_handler_exiting_failunknown_max_retries(self, mock_reactor, mock_do_recover):\n self.action_run.action_command = mock.create_autospec(\n actioncommand.ActionCommand,\n exit_status=None,\n )\n self.action_run.machine.transition(\"start\")\n self.action_run.machine.transition(\"started\")\n\n def exit_unknown(*args, **kwargs):\n self.action_run.handler(\n self.action_run.action_command,\n ActionCommand.EXITING,\n )\n\n # Each time do_recover is called, end up exiting unknown again\n mock_do_recover.side_effect = exit_unknown\n\n # Start the cycle\n exit_unknown()\n\n assert mock_do_recover.call_count == MAX_RECOVER_TRIES\n last_call = mock_do_recover.call_args\n expected_delay = INITIAL_RECOVER_DELAY * (3 ** (MAX_RECOVER_TRIES - 1))\n assert last_call == mock.call(self.action_run, delay=expected_delay)\n\n assert self.action_run.is_unknown\n assert self.action_run.exit_status is None\n assert self.action_run.end_time is not None\n\n\nclass TestActionRunStateRestore:\n now = datetime.datetime(2012, 3, 14, 15, 19)\n\n @pytest.fixture(autouse=True)\n def setup_action_run(self, mock_current_time):\n self.parent_context = {}\n self.output_path = [\"one\", \"two\"]\n self.run_node = MagicMock()\n mock_current_time.return_value = self.now\n self.command_config = ActionCommandConfig(\n command=\"do {actionname}\",\n cpus=1,\n )\n self.action_config = mock.Mock(command_config=self.command_config)\n self.action_graph = actiongraph.ActionGraph(\n {\"theaction\": self.action_config},\n {\"theaction\": set()},\n {\"theaction\": set()},\n )\n\n @pytest.fixture\n def state_data(self):\n # State data with command config and retries.\n yield {\n \"job_run_id\": \"theid\",\n \"action_name\": \"theaction\",\n \"node_name\": \"anode\",\n \"run_time\": \"the_run_time\",\n \"start_time\": \"start_time\",\n \"end_time\": \"end\",\n \"exit_status\": 0,\n \"attempts\": [\n dict(\n command_config=self.command_config.state_data,\n rendered_command=\"do theaction\",\n start_time=\"start\",\n end_time=\"end\",\n exit_status=0,\n ),\n ],\n \"state\": \"succeeded\",\n }\n\n @pytest.fixture\n def state_data_old(self):\n # State data before command config and retries are separate.\n yield {\n \"job_run_id\": \"theid\",\n \"action_name\": \"theaction\",\n \"node_name\": \"anode\",\n \"command\": \"do {actionname}\",\n \"start_time\": \"start_time\",\n \"end_time\": \"end\",\n \"state\": \"succeeded\",\n }\n\n def test_from_state_old(self, state_data_old):\n state_data = state_data_old\n action_run = ActionRun.from_state(\n state_data,\n self.parent_context,\n list(self.output_path),\n self.run_node,\n self.action_graph,\n )\n\n for key, value in state_data.items():\n if key in [\"state\", \"node_name\"]:\n continue\n assert getattr(action_run, key) == value\n\n assert action_run.is_succeeded\n assert not action_run.is_cleanup\n assert action_run.output_path[:2] == self.output_path\n assert action_run.command_config.command == state_data[\"command\"]\n assert action_run.command == state_data[\"command\"]\n\n def test_from_state_old_with_mesos_task_id(self, state_data_old):\n state_data = state_data_old\n state_data[\"mesos_task_id\"] = \"task\"\n action_run = ActionRun.from_state(\n state_data,\n self.parent_context,\n list(self.output_path),\n self.run_node,\n self.action_graph,\n )\n\n for key, value in state_data.items():\n if key in [\"state\", \"node_name\", \"mesos_task_id\"]:\n continue\n assert getattr(action_run, key) == value\n\n assert action_run.is_succeeded\n assert action_run.last_attempt.mesos_task_id == state_data[\"mesos_task_id\"]\n\n def test_from_state_old_not_started(self, state_data_old):\n state_data = state_data_old\n state_data[\"start_time\"] = None\n state_data[\"state\"] = \"scheduled\"\n action_run = ActionRun.from_state(\n state_data,\n self.parent_context,\n list(self.output_path),\n self.run_node,\n self.action_graph,\n )\n\n for key, value in state_data.items():\n if key in [\"state\", \"node_name\"]:\n continue\n assert getattr(action_run, key) == value\n\n assert action_run.is_scheduled\n assert action_run.exit_statuses == []\n assert len(action_run.attempts) == 0\n\n def test_from_state_old_rendered_and_exited(self, state_data_old):\n state_data = state_data_old\n state_data[\"rendered_command\"] = \"do things theaction\"\n state_data[\"exit_status\"] = 0\n action_run = ActionRun.from_state(\n state_data,\n self.parent_context,\n list(self.output_path),\n self.run_node,\n self.action_graph,\n )\n\n for key, value in state_data.items():\n if key in [\"state\", \"node_name\", \"command\", \"rendered_command\"]:\n continue\n assert getattr(action_run, key) == value\n\n assert action_run.is_succeeded\n assert action_run.exit_statuses == [0]\n assert action_run.command_config.command == state_data[\"command\"]\n assert action_run.command == state_data[\"rendered_command\"]\n\n def test_from_state_old_retries(self, state_data_old):\n state_data = state_data_old\n state_data[\"rendered_command\"] = \"do things theaction\"\n state_data[\"exit_status\"] = 0\n state_data[\"exit_statuses\"] = [1]\n action_run = ActionRun.from_state(\n state_data,\n self.parent_context,\n list(self.output_path),\n self.run_node,\n self.action_graph,\n )\n\n for key, value in state_data.items():\n if key in [\n \"state\",\n \"node_name\",\n \"command\",\n \"rendered_command\",\n \"exit_statuses\",\n ]:\n continue\n assert getattr(action_run, key) == value\n\n assert action_run.is_succeeded\n assert action_run.exit_statuses == [1, 0]\n assert len(action_run.attempts) == 2\n\n def test_from_state_running(self, state_data):\n state_data[\"state\"] = \"running\"\n action_run = ActionRun.from_state(\n state_data,\n self.parent_context,\n self.output_path,\n self.run_node,\n self.action_graph,\n lambda: None,\n )\n assert action_run.is_unknown\n\n def test_from_state_starting(self, state_data):\n state_data[\"state\"] = \"starting\"\n action_run = ActionRun.from_state(\n state_data,\n self.parent_context,\n self.output_path,\n self.run_node,\n self.action_graph,\n lambda: None,\n )\n assert action_run.is_unknown\n\n def test_from_state_queued(self, state_data):\n state_data[\"state\"] = \"queued\"\n action_run = ActionRun.from_state(\n state_data,\n self.parent_context,\n self.output_path,\n self.run_node,\n self.action_graph,\n lambda: None,\n )\n assert action_run.is_queued\n\n def test_from_state_no_node_name(self, state_data):\n del state_data[\"node_name\"]\n action_run = ActionRun.from_state(\n state_data,\n self.parent_context,\n self.output_path,\n self.run_node,\n self.action_graph,\n lambda: None,\n )\n assert action_run.node == self.run_node\n\n @mock.patch(\"tron.core.actionrun.node.NodePoolRepository\", autospec=True)\n def test_from_state_with_node_exists(self, mock_store, state_data):\n ActionRun.from_state(\n state_data,\n self.parent_context,\n self.output_path,\n self.run_node,\n self.action_graph,\n lambda: None,\n )\n mock_store.get_instance().get_node.assert_called_with(\n state_data[\"node_name\"],\n self.run_node,\n )\n\n def test_from_state_after_rendered_command(self, state_data):\n action_run = ActionRun.from_state(\n state_data,\n self.parent_context,\n self.output_path,\n self.run_node,\n self.action_graph,\n lambda: None,\n )\n assert action_run.command_config == self.command_config\n assert len(action_run.attempts) == len(state_data[\"attempts\"])\n assert action_run.exit_statuses == [0]\n assert action_run.command == state_data[\"attempts\"][-1][\"rendered_command\"]\n\n def test_from_state_action_config_gone(self, state_data):\n state_data[\"action_name\"] = \"old_action\"\n action_run = ActionRun.from_state(\n state_data,\n self.parent_context,\n self.output_path,\n self.run_node,\n self.action_graph,\n lambda: None,\n )\n assert action_run.command_config.command == \"\"\n assert action_run.command == state_data[\"attempts\"][-1][\"rendered_command\"]\n\n\nclass TestActionRunCollection:\n def _build_run(self, action):\n mock_node = mock.create_autospec(node.Node)\n return ActionRun(\n \"id\",\n action.name,\n mock_node,\n command_config=action.command_config,\n output_path=self.output_path,\n )\n\n @pytest.fixture(autouse=True)\n def setup_runs(self, output_path):\n action_names = [\"action_name\", \"second_name\", \"cleanup\"]\n\n actions = []\n for name in action_names:\n m = mock.Mock(\n name=name,\n required_actions=[],\n command_config=ActionCommandConfig(command=\"old\"),\n )\n m.name = name\n actions.append(m)\n\n self.action_graph = actiongraph.ActionGraph(\n {a.name: a for a in actions},\n {\"action_name\": set(), \"second_name\": set(), \"cleanup\": set()},\n {\"action_name\": set(), \"second_name\": set(), \"cleanup\": set()},\n )\n self.output_path = output_path\n self.command = \"do command\"\n self.action_runs = [self._build_run(action) for action in actions]\n self.run_map = {a.action_name: a for a in self.action_runs}\n self.run_map[\"cleanup\"].is_cleanup = True\n self.collection = ActionRunCollection(self.action_graph, self.run_map)\n\n def test__init__(self):\n assert self.collection.action_graph == self.action_graph\n assert self.collection.run_map == self.run_map\n assert self.collection.proxy_action_runs_with_cleanup\n\n def test_action_runs_for_actions(self):\n m = MagicMock()\n m.name = \"action_name\"\n actions = [m]\n action_runs = self.collection.action_runs_for_actions(actions)\n assert list(action_runs) == self.action_runs[:1]\n\n def test_get_action_runs_with_cleanup(self):\n runs = self.collection.get_action_runs_with_cleanup()\n assert set(runs) == set(self.action_runs)\n\n def test_get_action_runs(self):\n runs = self.collection.get_action_runs()\n assert set(runs) == set(self.action_runs[:2])\n\n def test_cleanup_action_run(self):\n assert self.action_runs[2] == self.collection.cleanup_action_run\n\n def test_update_action_config_no_changes(self):\n assert self.collection.update_action_config(self.action_graph) is False\n\n def test_update_action_config(self):\n # Latest config has 'new_name' instead of 'action_name'\n new_action_names = [\"new_name\", \"second_name\", \"cleanup\"]\n new_actions = []\n for name in new_action_names:\n action = mock.Mock(\n name=name,\n required_actions=[],\n command_config=ActionCommandConfig(command=\"new\"),\n )\n action.name = name\n new_actions.append(action)\n\n new_action_graph = actiongraph.ActionGraph(\n {a.name: a for a in new_actions},\n {\"new_name\": set(), \"second_name\": set(), \"cleanup\": set()},\n {\"new_name\": set(), \"second_name\": set(), \"cleanup\": set()},\n )\n assert self.collection.update_action_config(new_action_graph) is True\n assert self.collection.action_graph != new_action_graph\n\n updated_action_runs = self.collection.action_runs_with_cleanup\n # Action names should be unchanged\n assert sorted(run.name for run in updated_action_runs) == sorted(run.name for run in self.action_runs)\n\n for run in updated_action_runs:\n if run.name == \"action_name\":\n assert run.command_config.command == \"old\"\n else:\n assert run.command_config.command == \"new\"\n\n def test_state_data(self):\n state_data = self.collection.state_data\n assert_length(state_data, len(self.action_runs[:2]))\n\n def test_cleanup_action_state_data(self):\n state_data = self.collection.cleanup_action_state_data\n assert state_data[\"action_name\"] == \"cleanup\"\n\n def test_cleanup_action_state_data_no_cleanup_action(self):\n del self.collection.run_map[\"cleanup\"]\n assert not self.collection.cleanup_action_state_data\n\n def test_get_startable_action_runs(self):\n action_runs = self.collection.get_startable_action_runs()\n assert set(action_runs) == set(self.action_runs[:2])\n\n def test_get_startable_action_runs_none(self):\n self.collection.run_map.clear()\n action_runs = self.collection.get_startable_action_runs()\n assert set(action_runs) == set()\n\n def test_has_startable_action_runs(self):\n assert self.collection.has_startable_action_runs\n\n def test_has_startable_action_runs_false(self):\n self.collection.run_map.clear()\n assert not self.collection.has_startable_action_runs\n\n def test_is_complete_false(self):\n assert not self.collection.is_complete\n\n def test_is_complete_true(self):\n for action_run in self.collection.action_runs_with_cleanup:\n action_run.machine.state = ActionRun.SKIPPED\n assert self.collection.is_complete\n\n def test_is_done_false(self):\n assert not self.collection.is_done\n\n def test_is_done_false_because_of_running(self):\n action_run = self.collection.run_map[\"action_name\"]\n action_run.machine.state = ActionRun.RUNNING\n assert not self.collection.is_done\n\n def test_is_done_true_because_blocked(self):\n self.run_map[\"action_name\"].machine.state = ActionRun.FAILED\n self.run_map[\"second_name\"].machine.state = ActionRun.WAITING\n autospec_method(self.collection._is_run_blocked)\n\n self.collection._is_run_blocked.return_value = True\n assert self.collection.is_done\n assert self.collection.is_failed\n self.collection._is_run_blocked.assert_called_with(\n self.run_map[\"second_name\"],\n in_job_only=True,\n )\n\n def test_is_done_true(self):\n for action_run in self.collection.action_runs_with_cleanup:\n action_run.machine.state = ActionRun.FAILED\n assert self.collection.is_done\n\n def test_is_failed_false_not_done(self):\n self.run_map[\"action_name\"].machine.state = ActionRun.FAILED\n assert not self.collection.is_failed\n\n def test_is_failed_false_no_failed(self):\n for action_run in self.collection.action_runs_with_cleanup:\n action_run.machine.state = ActionRun.SUCCEEDED\n assert not self.collection.is_failed\n\n def test_is_failed_true(self):\n for action_run in self.collection.action_runs_with_cleanup:\n action_run.machine.state = ActionRun.FAILED\n assert self.collection.is_failed\n\n def test__getattr__(self):\n assert self.collection.is_scheduled\n assert not self.collection.is_cancelled\n assert not self.collection.is_running\n assert self.collection.ready()\n\n def test__str__(self):\n self.collection._is_run_blocked = lambda r: r.action_name != \"cleanup\"\n expected = [\n \"ActionRunCollection\",\n \"second_name(scheduled:blocked)\",\n \"action_name(scheduled:blocked)\",\n \"cleanup(scheduled)\",\n ]\n for expectation in expected:\n assert expectation in str(self.collection)\n\n def test_end_time(self):\n max_end_time = datetime.datetime(2013, 6, 15)\n self.run_map[\"action_name\"].machine.state = ActionRun.FAILED\n self.run_map[\"action_name\"].end_time = datetime.datetime(2013, 5, 12)\n self.run_map[\"second_name\"].machine.state = ActionRun.SUCCEEDED\n self.run_map[\"second_name\"].end_time = max_end_time\n assert self.collection.end_time == max_end_time\n\n def test_end_time_not_done(self):\n self.run_map[\"action_name\"].end_time = datetime.datetime(2013, 5, 12)\n self.run_map[\"action_name\"].machine.state = ActionRun.FAILED\n self.run_map[\"second_name\"].end_time = None\n self.run_map[\"second_name\"].machine.state = ActionRun.RUNNING\n assert self.collection.end_time is None\n\n def test_end_time_not_started(self):\n assert self.collection.end_time is None\n\n\nclass TestActionRunCollectionIsRunBlocked:\n def _build_run(self, name):\n mock_node = mock.create_autospec(node.Node)\n return ActionRun(\n \"id\",\n name,\n mock_node,\n self.command_config,\n output_path=self.output_path,\n )\n\n @pytest.fixture(autouse=True)\n def setup_collection(self, output_path):\n action_names = [\"action_name\", \"second_name\", \"cleanup\"]\n\n actions = []\n for name in action_names:\n m = MagicMock()\n m.name = name\n actions.append(m)\n\n self.second_act = actions[1]\n action_map = {a.name: a for a in actions}\n self.action_graph = actiongraph.ActionGraph(\n action_map,\n {\"action_name\": set(), \"second_name\": {\"action_name\"}, \"cleanup\": set()},\n {\"action_name\": set(), \"second_name\": set(), \"cleanup\": set()},\n )\n\n self.output_path = output_path\n self.command_config = ActionCommandConfig(command=\"do command\")\n self.action_runs = [self._build_run(name) for name in action_names]\n self.run_map = {a.action_name: a for a in self.action_runs}\n self.run_map[\"cleanup\"].is_cleanup = True\n self.collection = ActionRunCollection(self.action_graph, self.run_map)\n\n def test_is_run_blocked_no_required_actions(self):\n assert not self.collection._is_run_blocked(self.run_map[\"action_name\"])\n\n def test_is_run_blocked_completed_run(self):\n self.run_map[\"second_name\"].machine.state = ActionRun.FAILED\n assert not self.collection._is_run_blocked(self.run_map[\"second_name\"])\n\n self.run_map[\"second_name\"].machine.state = ActionRun.RUNNING\n assert not self.collection._is_run_blocked(self.run_map[\"second_name\"])\n\n def test_is_run_blocked_required_actions_completed(self):\n self.run_map[\"action_name\"].machine.state = ActionRun.SKIPPED\n assert not self.collection._is_run_blocked(self.run_map[\"second_name\"])\n\n def test_is_run_blocked_required_actions_blocked(self):\n third_act = MagicMock()\n third_act.name = \"third_act\"\n self.action_graph.action_map[\"third_act\"] = third_act\n self.action_graph.required_actions[\"third_act\"] = {self.second_act.name}\n self.run_map[\"third_act\"] = self._build_run(\"third_act\")\n\n self.run_map[\"action_name\"].machine.state = ActionRun.FAILED\n assert self.collection._is_run_blocked(self.run_map[\"third_act\"])\n\n def test_is_run_blocked_required_actions_scheduled(self):\n self.run_map[\"action_name\"].machine.state = ActionRun.SCHEDULED\n assert self.collection._is_run_blocked(self.run_map[\"second_name\"])\n\n def test_is_run_blocked_required_actions_starting(self):\n self.run_map[\"action_name\"].machine.state = ActionRun.STARTING\n assert self.collection._is_run_blocked(self.run_map[\"second_name\"])\n\n def test_is_run_blocked_required_actions_waiting(self):\n self.run_map[\"action_name\"].machine.state = ActionRun.WAITING\n assert self.collection._is_run_blocked(self.run_map[\"second_name\"])\n\n def test_is_run_blocked_required_actions_failed(self):\n self.run_map[\"action_name\"].machine.state = ActionRun.FAILED\n assert self.collection._is_run_blocked(self.run_map[\"second_name\"])\n\n def test_is_run_blocked_required_actions_missing(self):\n del self.run_map[\"action_name\"]\n assert not self.collection._is_run_blocked(self.run_map[\"second_name\"])\n\n def test_is_run_blocked_in_job_only(self):\n self.run_map[\"action_name\"].machine.state = ActionRun.SKIPPED\n self.run_map[\"second_name\"].triggered_by = [\"trigger\"]\n assert not self.collection._is_run_blocked(self.run_map[\"second_name\"], in_job_only=True)\n assert self.collection._is_run_blocked(self.run_map[\"second_name\"], in_job_only=False)\n\n\nclass TestMesosActionRun:\n @pytest.fixture(autouse=True)\n def setup_action_run(self):\n self.output_path = mock.MagicMock()\n self.command = \"do the command\"\n self.extra_volumes = [ConfigVolume(\"/mnt/foo\", \"/mnt/foo\", \"RO\")]\n self.constraints = [ConfigConstraint(\"an attr\", \"an op\", \"a val\")]\n self.docker_parameters = [ConfigParameter(\"init\", \"true\")]\n self.other_task_kwargs = {\n \"cpus\": 1,\n \"mem\": 50,\n \"disk\": 42,\n \"docker_image\": \"container:v2\",\n \"env\": {\n \"TESTING\": \"true\",\n \"TRON_JOB_NAMESPACE\": \"mynamespace\",\n \"TRON_JOB_NAME\": \"myjob\",\n \"TRON_RUN_NUM\": \"42\",\n \"TRON_ACTION\": \"action_name\",\n },\n }\n command_config = ActionCommandConfig(\n command=self.command,\n extra_volumes=self.extra_volumes,\n constraints=self.constraints,\n docker_parameters=self.docker_parameters,\n **self.other_task_kwargs,\n )\n self.action_run = MesosActionRun(\n job_run_id=\"mynamespace.myjob.42\",\n name=\"action_name\",\n command_config=command_config,\n node=mock.create_autospec(node.Node),\n output_path=self.output_path,\n executor=ExecutorTypes.mesos.value,\n )\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n @mock.patch(\"tron.core.actionrun.MesosClusterRepository\", autospec=True)\n def test_submit_command(self, mock_cluster_repo, mock_filehandler):\n serializer = mock_filehandler.OutputStreamSerializer.return_value\n # submit_command should add a new attempt\n self.action_run.attempts = [\n ActionRunAttempt(\n command_config=self.action_run.command_config,\n rendered_command=self.command,\n mesos_task_id=\"last_attempt\",\n ),\n ]\n with mock.patch.object(\n self.action_run,\n \"watch\",\n autospec=True,\n ) as mock_watch:\n new_attempt = self.action_run.create_attempt()\n self.action_run.submit_command(new_attempt)\n\n mock_get_cluster = mock_cluster_repo.get_cluster\n mock_get_cluster.assert_called_once_with()\n\n mock_get_cluster.return_value.create_task.assert_called_once_with(\n action_run_id=self.action_run.id,\n command=self.command,\n serializer=serializer,\n task_id=None,\n extra_volumes=[e._asdict() for e in self.extra_volumes],\n constraints=[[\"an attr\", \"an op\", \"a val\"]],\n docker_parameters=[{\"key\": \"init\", \"value\": \"true\"}],\n **self.other_task_kwargs,\n )\n task = mock_get_cluster.return_value.create_task.return_value\n mock_get_cluster.return_value.submit.assert_called_once_with(task)\n mock_watch.assert_called_once_with(task)\n assert self.action_run.last_attempt.mesos_task_id == task.get_mesos_id.return_value\n\n mock_filehandler.OutputStreamSerializer.assert_called_with(\n self.action_run.output_path,\n )\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n @mock.patch(\"tron.core.actionrun.MesosClusterRepository\", autospec=True)\n def test_submit_command_task_none(\n self,\n mock_cluster_repo,\n mock_filehandler,\n ):\n # Task is None if Mesos is disabled\n mock_get_cluster = mock_cluster_repo.get_cluster\n mock_get_cluster.return_value.create_task.return_value = None\n new_attempt = self.action_run.create_attempt()\n self.action_run.submit_command(new_attempt)\n\n mock_get_cluster.assert_called_once_with()\n assert mock_get_cluster.return_value.submit.call_count == 0\n assert self.action_run.is_failed\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n @mock.patch(\"tron.core.actionrun.MesosClusterRepository\", autospec=True)\n def test_recover(self, mock_cluster_repo, mock_filehandler):\n self.action_run.machine.state = ActionRun.UNKNOWN\n self.action_run.end_time = 1000\n self.action_run.exit_status = 0\n last_attempt = self.action_run.create_attempt()\n last_attempt.mesos_task_id = \"my_mesos_id\"\n last_attempt.end_time = 1000\n last_attempt.exit_status = 0\n serializer = mock_filehandler.OutputStreamSerializer.return_value\n with mock.patch.object(\n self.action_run,\n \"watch\",\n autospec=True,\n ) as mock_watch:\n assert self.action_run.recover()\n\n mock_get_cluster = mock_cluster_repo.get_cluster\n mock_get_cluster.assert_called_once_with()\n mock_get_cluster.return_value.create_task.assert_called_once_with(\n action_run_id=self.action_run.id,\n command=self.command,\n serializer=serializer,\n task_id=\"my_mesos_id\",\n extra_volumes=[e._asdict() for e in self.extra_volumes],\n constraints=[[\"an attr\", \"an op\", \"a val\"]],\n docker_parameters=[{\"key\": \"init\", \"value\": \"true\"}],\n **self.other_task_kwargs,\n )\n task = mock_get_cluster.return_value.create_task.return_value\n mock_get_cluster.return_value.recover.assert_called_once_with(task)\n mock_watch.assert_called_once_with(task)\n\n assert self.action_run.is_running\n assert self.action_run.end_time is None\n assert self.action_run.exit_status is None\n assert last_attempt.end_time is None\n assert last_attempt.exit_status is None\n mock_filehandler.OutputStreamSerializer.assert_called_with(\n self.action_run.output_path,\n )\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n @mock.patch(\"tron.core.actionrun.MesosClusterRepository\", autospec=True)\n def test_recover_done_no_change(self, mock_cluster_repo, mock_filehandler):\n self.action_run.machine.state = ActionRun.SUCCEEDED\n last_attempt = self.action_run.create_attempt()\n last_attempt.mesos_task_id = \"my_mesos_id\"\n\n assert not self.action_run.recover()\n assert mock_cluster_repo.get_cluster.call_count == 0\n assert self.action_run.is_succeeded\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n @mock.patch(\"tron.core.actionrun.MesosClusterRepository\", autospec=True)\n def test_recover_no_mesos_task_id(\n self,\n mock_cluster_repo,\n mock_filehandler,\n ):\n self.action_run.machine.state = ActionRun.UNKNOWN\n last_attempt = self.action_run.create_attempt()\n last_attempt.mesos_task_id = None\n\n assert not self.action_run.recover()\n assert mock_cluster_repo.get_cluster.call_count == 0\n assert self.action_run.is_unknown\n assert self.action_run.end_time is not None\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n @mock.patch(\"tron.core.actionrun.MesosClusterRepository\", autospec=True)\n def test_recover_task_none(self, mock_cluster_repo, mock_filehandler):\n self.action_run.machine.state = ActionRun.UNKNOWN\n last_attempt = self.action_run.create_attempt()\n last_attempt.mesos_task_id = \"my_mesos_id\"\n # Task is None if Mesos is disabled\n mock_get_cluster = mock_cluster_repo.get_cluster\n mock_get_cluster.return_value.create_task.return_value = None\n assert not self.action_run.recover()\n\n mock_get_cluster.assert_called_once_with()\n assert self.action_run.is_unknown\n assert mock_get_cluster.return_value.recover.call_count == 0\n assert self.action_run.end_time is not None\n\n @mock.patch(\"tron.core.actionrun.MesosClusterRepository\", autospec=True)\n def test_kill_task(self, mock_cluster_repo):\n mock_get_cluster = mock_cluster_repo.get_cluster\n last_attempt = self.action_run.create_attempt()\n last_attempt.mesos_task_id = \"fake_task_id\"\n self.action_run.machine.state = ActionRun.RUNNING\n\n self.action_run.kill()\n mock_get_cluster.return_value.kill.assert_called_once_with(\n last_attempt.mesos_task_id,\n )\n\n @mock.patch(\"tron.core.actionrun.MesosClusterRepository\", autospec=True)\n def test_kill_task_no_task_id(self, mock_cluster_repo):\n self.action_run.machine.state = ActionRun.RUNNING\n self.action_run.create_attempt()\n error_message = self.action_run.kill()\n assert error_message == \"Error: Can't find task id for the action.\"\n\n @mock.patch(\"tron.core.actionrun.MesosClusterRepository\", autospec=True)\n def test_stop_task(self, mock_cluster_repo):\n mock_get_cluster = mock_cluster_repo.get_cluster\n last_attempt = self.action_run.create_attempt()\n last_attempt.mesos_task_id = \"fake_task_id\"\n self.action_run.machine.state = ActionRun.RUNNING\n\n self.action_run.stop()\n mock_get_cluster.return_value.kill.assert_called_once_with(\n last_attempt.mesos_task_id,\n )\n\n @mock.patch(\"tron.core.actionrun.MesosClusterRepository\", autospec=True)\n def test_stop_task_no_task_id(self, mock_cluster_repo):\n self.action_run.machine.state = ActionRun.RUNNING\n self.action_run.create_attempt()\n error_message = self.action_run.stop()\n assert error_message == \"Error: Can't find task id for the action.\"\n\n def test_handler_exiting_unknown(self):\n self.action_run.action_command = mock.create_autospec(\n actioncommand.ActionCommand,\n exit_status=None,\n )\n self.action_run.machine.transition(\"start\")\n self.action_run.machine.transition(\"started\")\n assert self.action_run.handler(\n self.action_run.action_command,\n ActionCommand.EXITING,\n )\n assert self.action_run.is_unknown\n assert self.action_run.exit_status is None\n assert self.action_run.end_time is not None\n\n def test_handler_exiting_unknown_retry(self):\n self.action_run.action_command = mock.create_autospec(\n actioncommand.ActionCommand,\n exit_status=None,\n )\n self.action_run.retries_remaining = 1\n self.action_run.start = mock.Mock()\n\n self.action_run.machine.transition(\"start\")\n self.action_run.machine.transition(\"started\")\n assert self.action_run.handler(\n self.action_run.action_command,\n ActionCommand.EXITING,\n )\n assert self.action_run.retries_remaining == 0\n assert not self.action_run.is_unknown\n assert self.action_run.start.call_count == 1\n\n def test_handler_exiting_failstart_failed(self):\n self.action_run.action_command = mock.create_autospec(\n actioncommand.ActionCommand,\n exit_status=1,\n )\n self.action_run.machine.transition(\"start\")\n assert self.action_run.handler(\n self.action_run.action_command,\n ActionCommand.FAILSTART,\n )\n assert self.action_run.is_failed\n\n\nclass TestKubernetesActionRun:\n @pytest.fixture\n def mock_k8s_action_run(self):\n command_config = ActionCommandConfig(\n command=\"mock_command\",\n extra_volumes=set(),\n constraints=set(),\n docker_parameters=set(),\n cpus=1,\n mem=50,\n disk=42,\n docker_image=\"container:v2\",\n env={\n \"TESTING\": \"true\",\n \"TRON_JOB_NAMESPACE\": \"mock_namespace\",\n \"TRON_JOB_NAME\": \"mock_job\",\n \"TRON_RUN_NUM\": \"42\",\n \"TRON_ACTION\": \"mock_action_name\",\n },\n labels={\n \"tron.yelp.com/run_num\": \"42\",\n },\n )\n\n return KubernetesActionRun(\n job_run_id=\"mock_namespace.mock_job.42\",\n name=\"mock_action_name\",\n command_config=command_config,\n node=mock.create_autospec(node.Node),\n output_path=mock.create_autospec(filehandler.OutputPath),\n executor=ExecutorTypes.kubernetes.value,\n )\n\n def test_k8s_handler_exiting_unknown(self, mock_k8s_action_run):\n mock_k8s_action_run.action_command = mock.create_autospec(\n actioncommand.ActionCommand,\n exit_status=None,\n )\n mock_k8s_action_run.machine.transition(\"start\")\n mock_k8s_action_run.machine.transition(\"started\")\n assert mock_k8s_action_run.handler(\n mock_k8s_action_run.action_command,\n ActionCommand.EXITING,\n )\n assert mock_k8s_action_run.is_unknown\n assert mock_k8s_action_run.exit_status is None\n assert mock_k8s_action_run.end_time is not None\n\n def test_handler_exiting_unknown_retry(self, mock_k8s_action_run):\n mock_k8s_action_run.action_command = mock.create_autospec(\n actioncommand.ActionCommand,\n exit_status=None,\n )\n mock_k8s_action_run.retries_remaining = 1\n mock_k8s_action_run.start = mock.Mock()\n\n mock_k8s_action_run.machine.transition(\"start\")\n mock_k8s_action_run.machine.transition(\"started\")\n assert mock_k8s_action_run.handler(\n mock_k8s_action_run.action_command,\n ActionCommand.EXITING,\n )\n assert mock_k8s_action_run.retries_remaining == 0\n assert not mock_k8s_action_run.is_unknown\n assert mock_k8s_action_run.start.call_count == 1\n\n def test_handler_exiting_failstart_failed(self, mock_k8s_action_run):\n mock_k8s_action_run.action_command = mock.create_autospec(\n actioncommand.ActionCommand,\n exit_status=1,\n )\n mock_k8s_action_run.machine.transition(\"start\")\n assert mock_k8s_action_run.handler(\n mock_k8s_action_run.action_command,\n ActionCommand.FAILSTART,\n )\n assert mock_k8s_action_run.is_failed\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n @mock.patch(\"tron.core.actionrun.KubernetesClusterRepository\", autospec=True)\n def test_recover(self, mock_cluster_repo, mock_filehandler, mock_k8s_action_run):\n mock_k8s_action_run.machine.state = ActionRun.UNKNOWN\n mock_k8s_action_run.end_time = 1000\n mock_k8s_action_run.exit_status = 0\n last_attempt = mock_k8s_action_run.create_attempt()\n last_attempt.kubernetes_task_id = \"test-k8s-task-id\"\n last_attempt.end_time = 1000\n last_attempt.exit_status = 0\n serializer = mock_filehandler.OutputStreamSerializer.return_value\n with mock.patch.object(\n mock_k8s_action_run,\n \"watch\",\n autospec=True,\n ) as mock_watch:\n assert mock_k8s_action_run.recover()\n\n mock_get_cluster = mock_cluster_repo.get_cluster\n mock_get_cluster.assert_called_once_with()\n mock_get_cluster.return_value.create_task.assert_called_once_with(\n action_run_id=mock_k8s_action_run.id,\n command=last_attempt.rendered_command,\n cpus=mock_k8s_action_run.command_config.cpus,\n mem=mock_k8s_action_run.command_config.mem,\n disk=mock_k8s_action_run.command_config.disk,\n docker_image=mock_k8s_action_run.command_config.docker_image,\n env=mock.ANY,\n secret_env=mock_k8s_action_run.command_config.secret_env,\n field_selector_env=mock_k8s_action_run.command_config.field_selector_env,\n serializer=serializer,\n volumes=mock_k8s_action_run.command_config.extra_volumes,\n secret_volumes=mock_k8s_action_run.command_config.secret_volumes,\n projected_sa_volumes=mock_k8s_action_run.command_config.projected_sa_volumes,\n cap_add=mock_k8s_action_run.command_config.cap_add,\n cap_drop=mock_k8s_action_run.command_config.cap_drop,\n task_id=last_attempt.kubernetes_task_id,\n node_selectors=mock_k8s_action_run.command_config.node_selectors,\n node_affinities=mock_k8s_action_run.command_config.node_affinities,\n topology_spread_constraints=mock_k8s_action_run.command_config.topology_spread_constraints,\n pod_labels={\n \"tron.yelp.com/run_num\": \"42\",\n \"tron.yelp.com/attempt_number\": \"0\",\n },\n pod_annotations=mock_k8s_action_run.command_config.annotations,\n service_account_name=mock_k8s_action_run.command_config.service_account_name,\n ports=mock_k8s_action_run.command_config.ports,\n )\n task = mock_get_cluster.return_value.create_task.return_value\n mock_get_cluster.return_value.recover.assert_called_once_with(task)\n mock_watch.assert_called_once_with(task)\n\n assert mock_k8s_action_run.is_running\n assert mock_k8s_action_run.end_time is None\n assert mock_k8s_action_run.exit_status is None\n assert last_attempt.end_time is None\n assert last_attempt.exit_status is None\n mock_filehandler.OutputStreamSerializer.assert_called_with(\n mock_k8s_action_run.output_path,\n )\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n @mock.patch(\"tron.core.actionrun.KubernetesClusterRepository\", autospec=True)\n def test_recover_done_no_change(\n self,\n mock_cluster_repo,\n mock_filehandler,\n mock_k8s_action_run,\n ):\n mock_k8s_action_run.machine.state = ActionRun.SUCCEEDED\n last_attempt = mock_k8s_action_run.create_attempt()\n last_attempt.kubernetes_task_ic = \"test-kubernetes-task-id\"\n\n assert not mock_k8s_action_run.recover()\n assert mock_cluster_repo.get_cluster.call_count != 0\n assert mock_k8s_action_run.is_succeeded\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n @mock.patch(\"tron.core.actionrun.KubernetesClusterRepository\", autospec=True)\n def test_recover_no_k8s_task_id(\n self,\n mock_cluster_repo,\n mock_filehandler,\n mock_k8s_action_run,\n ):\n print(f\"cluster: {type(mock_cluster_repo)} filehand: {type(mock_filehandler)} ar: {type(mock_k8s_action_run)}\")\n mock_k8s_action_run.machine.state = ActionRun.UNKNOWN\n last_attempt = mock_k8s_action_run.create_attempt()\n last_attempt.mesos_task_id = None\n\n assert not mock_k8s_action_run.recover()\n assert mock_k8s_action_run.is_unknown\n assert mock_k8s_action_run.end_time is not None\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n @mock.patch(\"tron.core.actionrun.KubernetesClusterRepository\", autospec=True)\n def test_recover_task_none(self, mock_cluster_repo, mock_filehandler, mock_k8s_action_run):\n mock_k8s_action_run.machine.state = ActionRun.UNKNOWN\n last_attempt = mock_k8s_action_run.create_attempt()\n last_attempt.kubernetes_task_id = \"test-kubernetes-task-id\"\n # Task is None e.g. if Kubernetes is disabled\n mock_get_cluster = mock_cluster_repo.get_cluster\n mock_get_cluster.return_value.create_task.return_value = None\n assert not mock_k8s_action_run.recover()\n\n mock_get_cluster.assert_called_once_with()\n assert mock_k8s_action_run.is_unknown\n assert mock_get_cluster.return_value.recover.call_count == 0\n assert mock_k8s_action_run.end_time is not None\n\n @mock.patch(\"tron.core.actionrun.KubernetesClusterRepository\", autospec=True)\n def test_kill_task_k8s(self, mock_cluster_repo, mock_k8s_action_run):\n mock_get_cluster = mock_cluster_repo.get_cluster\n last_attempt = mock_k8s_action_run.create_attempt()\n last_attempt.kubernetes_task_id = \"fake_task_id\"\n mock_k8s_action_run.machine.state = ActionRun.RUNNING\n\n mock_k8s_action_run.kill()\n mock_get_cluster.return_value.kill.assert_called_once_with(last_attempt.kubernetes_task_id)\n\n @mock.patch(\"tron.core.actionrun.KubernetesClusterRepository\", autospec=True)\n def test_kill_task_no_task_id_k8s(self, mock_cluster_repo, mock_k8s_action_run):\n mock_k8s_action_run.machine.state = ActionRun.RUNNING\n mock_k8s_action_run.create_attempt()\n error_message = mock_k8s_action_run.kill()\n assert error_message == \"Error: Can't find task id for the action.\"\n\n @mock.patch(\"tron.core.actionrun.KubernetesClusterRepository\", autospec=True)\n def test_stop_task_k8s(self, mock_cluster_repo, mock_k8s_action_run):\n mock_get_cluster = mock_cluster_repo.get_cluster\n last_attempt = mock_k8s_action_run.create_attempt()\n last_attempt.kubernetes_task_id = \"fake_task_id\"\n mock_k8s_action_run.machine.state = ActionRun.RUNNING\n\n mock_k8s_action_run.stop()\n mock_get_cluster.return_value.kill.assert_called_once_with(last_attempt.kubernetes_task_id)\n\n @mock.patch(\"tron.core.actionrun.KubernetesClusterRepository\", autospec=True)\n def test_stop_task_no_task_id_k8s(self, mock_cluster_repo, mock_k8s_action_run):\n mock_k8s_action_run.machine.state = ActionRun.RUNNING\n mock_k8s_action_run.create_attempt()\n error_message = mock_k8s_action_run.stop()\n assert error_message == \"Error: Can't find task id for the action.\"\n\n @mock.patch(\"tron.core.actionrun.KubernetesClusterRepository\", autospec=True)\n def test_non_retryable_exit(self, mock_cluster_repo, mock_k8s_action_run):\n\n mock_cluster = mock.Mock()\n mock_cluster.non_retryable_exit_codes = [13]\n mock_cluster_repo.get_cluster.return_value = mock_cluster\n\n mock_k8s_action_run.action_command = mock.create_autospec(\n actioncommand.ActionCommand,\n exit_status=13,\n )\n mock_k8s_action_run.retries_remaining = 5\n\n mock_k8s_action_run.machine.transition(\"start\")\n mock_k8s_action_run.machine.transition(\"started\")\n assert mock_k8s_action_run.handler(\n mock_k8s_action_run.action_command,\n ActionCommand.EXITING,\n )\n\n assert mock_k8s_action_run.retries_remaining == 0\n assert mock_k8s_action_run.is_unknown\n\n @mock.patch(\"tron.core.actionrun.KubernetesClusterRepository\", autospec=True)\n def test_retryable_exit(self, mock_cluster_repo, mock_k8s_action_run):\n\n mock_cluster = mock.Mock()\n mock_cluster.non_retryable_exit_codes = [-12]\n\n mock_cluster_repo.get_cluster.return_value = mock_cluster\n\n mock_k8s_action_run.retries_remaining = 5\n mock_k8s_action_run.start = mock.Mock()\n\n mock_k8s_action_run._exit_unsuccessful(13)\n\n assert mock_k8s_action_run.retries_remaining == 4\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n @mock.patch(\"tron.core.actionrun.KubernetesClusterRepository\", autospec=True)\n def test_submit_command_first_attempt_labels(self, mock_cluster_repo, mock_filehandler, mock_k8s_action_run):\n with mock.patch.object(mock_k8s_action_run, \"watch\", autospec=True):\n new_attempt = mock_k8s_action_run.create_attempt()\n mock_k8s_action_run.submit_command(new_attempt)\n\n create_task_kwargs = mock_cluster_repo.get_cluster.return_value.create_task.call_args[1]\n assert create_task_kwargs[\"pod_labels\"][\"tron.yelp.com/attempt_number\"] == \"0\"\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n @mock.patch(\"tron.core.actionrun.KubernetesClusterRepository\", autospec=True)\n def test_submit_command_retry_attempt_labels(self, mock_cluster_repo, mock_filehandler, mock_k8s_action_run):\n mock_k8s_action_run.attempts = [\n ActionRunAttempt(\n command_config=mock_k8s_action_run.command_config,\n rendered_command=\"mock_command\",\n ),\n ActionRunAttempt(\n command_config=mock_k8s_action_run.command_config,\n rendered_command=\"mock_command\",\n ),\n ]\n with mock.patch.object(mock_k8s_action_run, \"watch\", autospec=True):\n new_attempt = mock_k8s_action_run.create_attempt()\n mock_k8s_action_run.submit_command(new_attempt)\n\n create_task_kwargs = mock_cluster_repo.get_cluster.return_value.create_task.call_args[1]\n assert create_task_kwargs[\"pod_labels\"][\"tron.yelp.com/attempt_number\"] == \"2\"\n\n @mock.patch(\"tron.core.actionrun.filehandler\", autospec=True)\n @mock.patch(\"tron.core.actionrun.KubernetesClusterRepository\", autospec=True)\n def test_recover_retry_attempt_labels(self, mock_cluster_repo, mock_filehandler, mock_k8s_action_run):\n mock_k8s_action_run.attempts = [\n ActionRunAttempt(\n command_config=mock_k8s_action_run.command_config,\n rendered_command=\"mock_command\",\n ),\n ActionRunAttempt(\n command_config=mock_k8s_action_run.command_config,\n rendered_command=\"mock_command\",\n ),\n ]\n mock_k8s_action_run.machine.state = ActionRun.UNKNOWN\n last_attempt = mock_k8s_action_run.create_attempt()\n last_attempt.kubernetes_task_id = \"test-k8s-task-id\"\n with mock.patch.object(mock_k8s_action_run, \"watch\", autospec=True):\n assert mock_k8s_action_run.recover()\n\n create_task_kwargs = mock_cluster_repo.get_cluster.return_value.create_task.call_args[1]\n assert create_task_kwargs[\"pod_labels\"][\"tron.yelp.com/attempt_number\"] == \"2\"\n" }, { "path": "tests/core/job_collection_test.py", "content": "from unittest import mock\n\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tests.testingutils import autospec_method\nfrom tron.core.job import Job\nfrom tron.core.job_collection import JobCollection\nfrom tron.core.job_scheduler import JobScheduler\nfrom tron.core.job_scheduler import JobSchedulerFactory\n\n\nclass TestJobCollection(TestCase):\n @setup\n def setup_collection(self):\n self.collection = JobCollection()\n\n def test_update_from_config(self):\n autospec_method(self.collection.jobs.filter_by_name)\n autospec_method(self.collection.add)\n factory = mock.create_autospec(JobSchedulerFactory)\n job_configs = {\"a\": mock.Mock(), \"b\": mock.Mock()}\n result = self.collection.update_from_config(job_configs, factory, True)\n result = list(result)\n assert len(result) == len(job_configs)\n self.collection.jobs.filter_by_name.assert_called_with(job_configs)\n expected_calls = [mock.call(v) for v in job_configs.values()]\n assert factory.build.call_args_list == expected_calls\n assert self.collection.add.call_count == 2\n job_schedulers = [call[1][0] for call in self.collection.add.mock_calls[::2]]\n for job_scheduler in job_schedulers:\n job_scheduler.schedule.assert_called_with()\n job_scheduler.get_job.assert_called_with()\n\n def test_update_from_config_reconfigure_one_namespace(self):\n autospec_method(self.collection.jobs.filter_by_name)\n autospec_method(self.collection.add)\n factory = mock.create_autospec(JobSchedulerFactory)\n job_configs = {\n \"a.foo\": mock.Mock(namespace=\"a\"),\n \"b.foo\": mock.Mock(namespace=\"b\"),\n }\n result = self.collection.update_from_config(job_configs, factory, True, namespace_to_reconfigure=\"a\")\n result = list(result)\n assert len(result) == 1\n self.collection.jobs.filter_by_name.assert_called_with(job_configs)\n expected_calls = [mock.call(job_configs[\"a.foo\"])]\n assert factory.build.call_args_list == expected_calls\n assert self.collection.add.call_count == 1\n job_schedulers = [call[1][0] for call in self.collection.add.mock_calls[::2]]\n for job_scheduler in job_schedulers:\n job_scheduler.schedule.assert_called_with()\n job_scheduler.get_job.assert_called_with()\n\n def test_move_running_job(self):\n with mock.patch(\n \"tron.core.job_collection.JobCollection.get_by_name\",\n autospec=None,\n ) as mock_scheduler:\n mock_scheduler.return_value.get_job.return_value.status = Job.STATUS_RUNNING\n result = self.collection.move(\"old.test\", \"new.test\")\n assert \"Job is still running.\" in result\n\n def test_move(self):\n with mock.patch(\n \"tron.core.job_collection.JobCollection.get_by_name\",\n autospec=None,\n ) as mock_scheduler:\n mock_scheduler.return_value.get_job.return_value.status = Job.STATUS_ENABLED\n mock_scheduler.get_name.return_value = \"old.test\"\n self.collection.add(mock_scheduler)\n result = self.collection.move(\"old.test\", \"new.test\")\n assert \"succeeded\" in result\n\n def test_update(self):\n mock_scheduler = mock.create_autospec(JobScheduler)\n existing_scheduler = mock.create_autospec(JobScheduler)\n autospec_method(\n self.collection.get_by_name,\n return_value=existing_scheduler,\n )\n assert self.collection.update(mock_scheduler)\n self.collection.get_by_name.assert_called_with(\n mock_scheduler.get_name(),\n )\n existing_scheduler.update_from_job_scheduler.assert_called_with(\n mock_scheduler,\n )\n existing_scheduler.schedule_reconfigured.assert_called_with()\n" }, { "path": "tests/core/job_scheduler_test.py", "content": "import datetime\nfrom unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tests import testingutils\nfrom tests.assertions import assert_length\nfrom tron import actioncommand\nfrom tron.core import job\nfrom tron.core.actionrun import ActionRun\nfrom tron.core.job_scheduler import JobScheduler\nfrom tron.core.job_scheduler import JobSchedulerFactory\n\n\nclass TestJobSchedulerGetRunsToSchedule(TestCase):\n @setup\n def setup_job(self):\n self.scheduler = mock.Mock()\n run_collection = mock.Mock(has_pending=False)\n node_pool = mock.Mock()\n self.job = job.Job(\n \"jobname\",\n self.scheduler,\n run_collection=run_collection,\n node_pool=node_pool,\n )\n self.job_scheduler = JobScheduler(self.job)\n self.job.runs.get_pending.return_value = False\n self.scheduler.queue_overlapping = True\n\n def test_get_runs_to_schedule_with_pending(self):\n self.scheduler.queue_overlapping = False\n self.job.runs.has_pending = True\n job_runs = self.job_scheduler.get_runs_to_schedule(None)\n assert_length(job_runs, 0)\n\n def test_get_runs_to_schedule_guess(self):\n job_runs = list(self.job_scheduler.get_runs_to_schedule(None))\n\n assert self.job.scheduler.next_run_time.call_args_list == [mock.call(None)]\n assert_length(job_runs, 1)\n # This should return a JobRun which has the job attached as an observer\n job_runs[0].attach.assert_any_call(True, self.job)\n\n def test_get_runs_to_schedule_given(self):\n now = datetime.datetime.now()\n job_runs = list(self.job_scheduler.get_runs_to_schedule(now))\n\n assert self.job.scheduler.next_run_time.call_count == 0\n assert_length(job_runs, 1)\n # This should return a JobRun which has the job attached as an observer\n job_runs[0].attach.assert_any_call(True, self.job)\n\n\nclass JobSchedulerManualStartTestCase(testingutils.MockTimeTestCase):\n\n now = datetime.datetime.now()\n\n @setup\n def setup_job(self):\n self.scheduler = mock.Mock()\n run_collection = mock.Mock()\n node_pool = mock.Mock()\n self.job = job.Job(\n \"jobname\",\n self.scheduler,\n run_collection=run_collection,\n node_pool=node_pool,\n )\n self.job_scheduler = JobScheduler(self.job)\n self.manual_run = mock.Mock()\n self.job.build_new_runs = mock.Mock(return_value=[self.manual_run])\n\n def test_manual_start(self):\n manual_runs = self.job_scheduler.manual_start()\n\n self.job.build_new_runs.assert_called_with(self.now, manual=True)\n assert_length(manual_runs, 1)\n self.manual_run.start.assert_called_once_with()\n\n def test_manual_start_default_with_timezone(self):\n self.job.time_zone = mock.Mock()\n with mock.patch(\n \"tron.core.job_scheduler.timeutils.current_time\",\n autospec=True,\n ) as mock_current:\n manual_runs = self.job_scheduler.manual_start()\n mock_current.assert_called_with(tz=self.job.time_zone)\n self.job.build_new_runs.assert_called_with(\n mock_current.return_value,\n manual=True,\n )\n assert_length(manual_runs, 1)\n self.manual_run.start.assert_called_once_with()\n\n def test_manual_start_with_run_time(self):\n run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)\n manual_runs = self.job_scheduler.manual_start(run_time)\n\n self.job.build_new_runs.assert_called_with(run_time, manual=True)\n assert_length(manual_runs, 1)\n self.manual_run.start.assert_called_once_with()\n\n\nclass TestJobSchedulerSchedule(TestCase):\n @setup\n def setup_job(self):\n self.scheduler = mock.Mock(autospec=True)\n self.scheduler.next_run_time.return_value = 0\n mock_run = mock.Mock()\n mock_run.seconds_until_run_time.return_value = 0\n run_collection = mock.Mock(\n has_pending=False,\n autospec=True,\n return_value=[mock_run],\n )\n mock_build_new_run = mock.Mock()\n run_collection.build_new_run.return_value = mock_build_new_run\n mock_build_new_run.seconds_until_run_time.return_value = 0\n node_pool = mock.Mock()\n self.job = job.Job(\n name=\"jobname\",\n scheduler=self.scheduler,\n run_collection=run_collection,\n node_pool=node_pool,\n )\n self.job_scheduler = JobScheduler(self.job)\n self.original_build_new_runs = self.job.build_new_runs\n self.job.build_new_runs = mock.Mock(return_value=[mock_run])\n\n @mock.patch(\"tron.core.job_scheduler.reactor\", autospec=True)\n def test_enable(self, reactor):\n self.job.enabled = False\n self.job_scheduler.enable()\n assert self.job.enabled\n assert_length(reactor.callLater.mock_calls, 1)\n\n @mock.patch(\"tron.core.job_scheduler.reactor\", autospec=True)\n def test_enable_noop(self, reactor):\n self.job.enabled = True\n self.job_scheduler.enable()\n assert self.job.enabled\n assert_length(reactor.callLater.mock_calls, 0)\n\n @mock.patch(\"tron.core.job_scheduler.reactor\", autospec=True)\n def test_schedule(self, reactor):\n self.job.build_new_runs = self.original_build_new_runs\n self.job_scheduler.schedule()\n assert reactor.callLater.call_count == 1\n\n # Args passed to callLater\n call_args = reactor.callLater.mock_calls[0][1]\n assert_equal(call_args[1], self.job_scheduler.run_job)\n secs = call_args[0]\n run = call_args[2]\n\n run.seconds_until_run_time.assert_called_with()\n # Assert that we use the seconds we get from the run to schedule\n assert_equal(run.seconds_until_run_time.return_value, secs)\n\n @mock.patch(\"tron.core.job_scheduler.reactor\", autospec=True)\n def test_schedule_disabled_job(self, reactor):\n self.job.enabled = False\n self.job_scheduler.schedule()\n assert reactor.callLater.call_count == 0\n\n @mock.patch(\"tron.core.job_scheduler.reactor\", autospec=True)\n def test_handle_job_events_no_schedule_on_complete(self, reactor):\n self.job_scheduler.run_job = mock.Mock()\n self.job.scheduler.schedule_on_complete = False\n queued_job_run = mock.Mock()\n self.job.runs.get_first_queued = lambda: queued_job_run\n self.job_scheduler.handle_job_events(self.job, job.Job.NOTIFY_RUN_DONE)\n reactor.callLater.assert_any_call(\n 0,\n self.job_scheduler.run_job,\n queued_job_run,\n run_queued=True,\n )\n\n def test_handle_job_events_schedule_on_complete(self):\n self.job_scheduler.schedule = mock.Mock()\n self.job.scheduler.schedule_on_complete = True\n self.job_scheduler.handle_job_events(self.job, job.Job.NOTIFY_RUN_DONE)\n self.job_scheduler.schedule.assert_called_with()\n\n def test_handler_unknown_event(self):\n self.job.runs.get_runs_by_state = mock.Mock()\n self.job_scheduler.handler(self.job, \"some_other_event\")\n self.job.runs.get_runs_by_state.assert_not_called()\n\n def test_handler_no_queued(self):\n self.job_scheduler.run_job = mock.Mock()\n\n def get_queued(state):\n if state == ActionRun.QUEUED:\n return []\n\n self.job.runs.get_runs_by_state = get_queued\n self.job_scheduler.handler(self.job, job.Job.NOTIFY_RUN_DONE)\n self.job_scheduler.run_job.assert_not_called()\n\n @mock.patch(\"tron.core.job_scheduler.reactor\", autospec=True)\n def test_run_queue_schedule(self, reactor):\n with mock.patch.object(\n self.job_scheduler,\n \"schedule\",\n ) as mock_schedule:\n self.job_scheduler.run_job = mock.Mock()\n self.job.scheduler.schedule_on_complete = False\n queued_job_run = mock.Mock()\n self.job.runs.get_first_queued = lambda: queued_job_run\n self.job_scheduler.run_queue_schedule()\n reactor.callLater.assert_called_once_with(\n 0,\n self.job_scheduler.run_job,\n queued_job_run,\n run_queued=True,\n )\n mock_schedule.assert_called_once_with()\n\n\nclass TestJobSchedulerOther(TestCase):\n \"\"\"Test other JobScheduler functions\"\"\"\n\n def _make_job_scheduler(self, job_name, enabled=True):\n scheduler = mock.Mock()\n run_collection = mock.Mock()\n node_pool = mock.Mock()\n new_job = job.Job(\n job_name,\n scheduler,\n run_collection=run_collection,\n node_pool=node_pool,\n enabled=enabled,\n )\n return new_job, JobScheduler(new_job)\n\n @setup\n def setup_job(self):\n self.job, self.job_scheduler = self._make_job_scheduler(\n \"jobname\",\n True,\n )\n\n def test_disable(self):\n self.job.runs.cancel_pending = mock.Mock()\n\n self.job_scheduler.disable()\n\n assert not self.job.enabled\n assert self.job.runs.cancel_pending.call_count == 1\n\n def test_update_from_job_scheduler_disable(self):\n new_job, new_job_scheduler = self._make_job_scheduler(\"jobname\", False)\n self.job.update_from_job = mock.Mock()\n self.job_scheduler.disable = mock.Mock()\n\n self.job_scheduler.update_from_job_scheduler(new_job_scheduler)\n\n assert self.job.update_from_job.call_args == mock.call(\n new_job_scheduler.get_job(),\n )\n assert self.job_scheduler.disable.call_count == 1\n\n def test_update_from_job_scheduler_enable(self):\n new_job, new_job_scheduler = self._make_job_scheduler(\"jobname\", True)\n self.job.update_from_job = mock.Mock()\n self.job.enabled = False\n self.job.config_enabled = False\n self.job_scheduler.enable = mock.Mock()\n\n self.job_scheduler.update_from_job_scheduler(new_job_scheduler)\n\n assert self.job.update_from_job.call_args == mock.call(\n new_job_scheduler.get_job(),\n )\n assert self.job_scheduler.enable.call_count == 1\n\n def test_update_from_job_scheduler_no_config_change(self):\n new_job, new_job_scheduler = self._make_job_scheduler(\"jobname\", True)\n self.job.enabled = False\n self.job.update_from_job = mock.Mock()\n self.job_scheduler.enable = mock.Mock()\n self.job_scheduler.disable = mock.Mock()\n\n self.job_scheduler.update_from_job_scheduler(new_job_scheduler)\n\n assert self.job.update_from_job.call_args == mock.call(\n new_job_scheduler.get_job(),\n )\n assert self.job_scheduler.enable.call_count == 0\n assert self.job_scheduler.disable.call_count == 0\n assert self.job.config_enabled == new_job.config_enabled\n assert not self.job.enabled\n\n\nclass TestJobSchedulerFactory(TestCase):\n @setup\n def setup_factory(self):\n self.context = mock.Mock()\n self.output_stream_dir = mock.Mock()\n self.time_zone = mock.Mock()\n self.action_runner = mock.create_autospec(\n actioncommand.SubprocessActionRunnerFactory,\n )\n self.factory = JobSchedulerFactory(\n self.context,\n self.output_stream_dir,\n self.time_zone,\n self.action_runner,\n mock.Mock(),\n )\n\n def test_build(self):\n config = mock.Mock()\n with mock.patch(\n \"tron.core.job_scheduler.Job\",\n autospec=True,\n ) as mock_job:\n job_scheduler = self.factory.build(config)\n _, kwargs = mock_job.from_config.call_args\n assert_equal(kwargs[\"job_config\"], config)\n assert_equal(\n job_scheduler.get_job(),\n mock_job.from_config.return_value,\n )\n assert_equal(kwargs[\"parent_context\"], self.context)\n assert_equal(kwargs[\"output_path\"].base, self.output_stream_dir)\n assert_equal(kwargs[\"action_runner\"], self.action_runner)\n" }, { "path": "tests/core/job_test.py", "content": "import collections\nimport datetime\nfrom unittest import mock\nfrom unittest.mock import MagicMock\n\nimport pytest\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_not_equal\nfrom tests.assertions import assert_call\nfrom tests.assertions import assert_length\nfrom tests.testingutils import autospec_method\nfrom tron import actioncommand\nfrom tron import node\nfrom tron.core import job\nfrom tron.core import jobrun\nfrom tron.core.actionrun import ActionRun\nfrom tron.core.job_scheduler import JobScheduler\n\n\n@pytest.fixture\ndef mock_node_repo():\n with mock.patch(\n \"tron.core.job.node.NodePoolRepository\",\n autospec=True,\n ) as mock_node_repo:\n yield mock_node_repo\n\n\n@pytest.fixture\ndef mock_job(mock_node_repo):\n action_graph = mock.Mock(names=lambda: [\"one\", \"two\"])\n scheduler = mock.Mock()\n run_collection = MagicMock()\n nodes = mock.create_autospec(node.NodePool)\n mock_job = job.Job(\n \"jobname\",\n scheduler,\n run_collection=run_collection,\n action_graph=action_graph,\n node_pool=nodes,\n action_runner=actioncommand.NoActionRunnerFactory,\n )\n yield mock_job\n\n\nclass TestJob:\n @pytest.fixture(autouse=True)\n def setup_job(self, mock_job):\n self.job = mock_job\n autospec_method(self.job.notify)\n autospec_method(self.job.watch)\n yield\n\n def test__init__(self):\n assert str(self.job.output_path).endswith(self.job.name)\n\n def test_from_config(self, mock_node_repo):\n action = mock.MagicMock(\n name=\"first\",\n command=\"doit\",\n node=None,\n requires=[],\n )\n job_config = mock.Mock(\n node=\"thenodepool\",\n monitoring={\n \"team\": \"foo\",\n \"page\": True,\n },\n all_nodes=False,\n queueing=True,\n enabled=True,\n run_limit=20,\n actions={action.name: action},\n cleanup_action=None,\n )\n job_config.name = \"ajob\" # set this after mock creation to give it a \"real\" name attribute\n scheduler = \"scheduler_token\"\n parent_context = \"parent_context_token\"\n output_path = [\"base_path\"]\n mock_action_runner = mock.create_autospec(\n actioncommand.SubprocessActionRunnerFactory,\n )\n new_job = job.Job.from_config(\n job_config,\n scheduler,\n parent_context=parent_context,\n output_path=output_path,\n action_runner=mock_action_runner,\n action_graph=mock.Mock(),\n )\n\n assert_equal(new_job.scheduler, scheduler)\n assert_equal(new_job.context.next, parent_context)\n mock_node_repo.get_instance().get_by_name.assert_called_with(\n job_config.node,\n )\n assert_equal(new_job.enabled, True)\n assert_equal(new_job.get_monitoring()[\"team\"], \"foo\")\n assert new_job.action_graph\n\n def test_update_from_job(self):\n action_runner = mock.Mock()\n other_job = job.Job(\n \"otherjob\",\n \"scheduler\",\n action_runner=action_runner,\n run_limit=10,\n )\n self.job.update_from_job(other_job)\n assert_equal(self.job.name, \"otherjob\")\n assert_equal(self.job.scheduler, \"scheduler\")\n assert_equal(self.job, other_job)\n assert_equal(self.job.runs.run_limit, 10)\n\n def test_status_disabled(self):\n self.job.enabled = False\n assert_equal(self.job.status, self.job.STATUS_DISABLED)\n\n def test_status_enabled(self):\n self.job.runs.get_run_by_state = lambda state: MagicMock() if state == ActionRun.SCHEDULED else None\n self.job.runs.get_active.return_value = []\n assert_equal(self.job.status, self.job.STATUS_ENABLED)\n\n def test_status_running(self):\n self.job.runs.get_active.return_value = [MagicMock()]\n assert_equal(self.job.status, self.job.STATUS_RUNNING)\n\n def test_status_unknown(self):\n self.job.runs.get_active.return_value = []\n self.job.runs.get_run_by_state = lambda s: None\n assert_equal(self.job.status, self.job.STATUS_UNKNOWN)\n\n def test_state_data(self):\n state_data = self.job.state_data\n assert_equal(state_data[\"run_nums\"], self.job.runs.get_run_nums.return_value)\n assert state_data[\"enabled\"]\n\n def test_get_job_runs_from_state(self):\n job_runs = [\n dict(\n run_num=i,\n job_name=\"thename\",\n run_time=\"sometime\",\n start_time=\"start_time\",\n end_time=\"sometime\",\n cleanup_run=None,\n runs=[],\n )\n for i in range(0, 3)\n ]\n state_data = {\"enabled\": False, \"runs\": job_runs}\n self.job.get_job_runs_from_state(state_data)\n assert not self.job.enabled\n\n def test_build_new_runs(self):\n run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)\n runs = list(self.job.build_new_runs(run_time))\n\n self.job.node_pool.next.assert_called_with()\n node = self.job.node_pool.next.return_value\n assert_call(\n self.job.runs.build_new_run,\n 0,\n self.job,\n run_time,\n node,\n manual=False,\n )\n assert_length(runs, 1)\n self.job.watch.assert_called_with(runs[0])\n\n def test_build_new_runs_all_nodes(self):\n self.job.all_nodes = True\n run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)\n node_count = 2\n self.job.node_pool.nodes = [mock.Mock()] * node_count\n runs = list(self.job.build_new_runs(run_time))\n\n assert_length(runs, node_count)\n for i in range(len(runs)):\n node = self.job.node_pool.nodes[i]\n assert_call(\n self.job.runs.build_new_run,\n i,\n self.job,\n run_time,\n node,\n manual=False,\n )\n\n calls = []\n for r in runs:\n calls.extend(r.mock_calls)\n self.job.watch.assert_has_calls(calls)\n\n def test_build_new_runs_manual(self):\n run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)\n runs = list(self.job.build_new_runs(run_time, manual=True))\n\n self.job.node_pool.next.assert_called_with()\n node = self.job.node_pool.next.return_value\n assert_length(runs, 1)\n assert_call(\n self.job.runs.build_new_run,\n 0,\n self.job,\n run_time,\n node,\n manual=True,\n )\n self.job.watch.assert_called_with(runs[0])\n\n def test_handler(self):\n self.job.handler(None, jobrun.JobRun.NOTIFY_STATE_CHANGED)\n self.job.notify.assert_called_with(self.job.NOTIFY_STATE_CHANGE)\n\n self.job.handler(None, jobrun.JobRun.NOTIFY_DONE)\n self.job.notify.assert_called_with(self.job.NOTIFY_RUN_DONE)\n\n def test__eq__(self):\n other_job = job.Job(\"jobname\", \"scheduler\", run_collection=MagicMock())\n assert not self.job == other_job\n other_job.update_from_job(self.job)\n assert_equal(self.job, other_job)\n\n def test__ne__(self):\n other_job = job.Job(\"jobname\", \"scheduler\", run_collection=MagicMock())\n assert self.job != other_job\n other_job.update_from_job(self.job)\n assert not self.job != other_job\n\n def test__eq__true(self):\n action_runner = mock.Mock()\n first = job.Job(\"jobname\", \"scheduler\", action_runner=action_runner)\n second = job.Job(\"jobname\", \"scheduler\", action_runner=action_runner)\n assert_equal(first, second)\n\n def test__eq__false(self):\n first = job.Job(\"jobname\", \"scheduler\", action_runner=mock.Mock())\n second = job.Job(\"jobname\", \"scheduler\", action_runner=mock.Mock())\n assert_not_equal(first, second)\n\n\ndef test_job_watch_notifies_about_runs(mock_job):\n # Separate from the above tests because we don't want\n # watch to be mocked here.\n new_run = jobrun.JobRun(\n job_name=\"test\",\n run_num=1,\n run_time=\"some_time\",\n node=\"node\",\n )\n with mock.patch.object(mock_job, \"handler\",) as mock_handler, mock.patch.object(\n mock_job,\n \"notify\",\n ) as mock_notify:\n mock_job.watch(new_run)\n\n # Make sure that the job is still watching correctly\n # by checking it handles events\n new_run.notify(\"test_event\", \"test_data\")\n assert mock_handler.call_args_list == [mock.call(new_run, \"test_event\", \"test_data\")]\n\n # Check that the job notifies its watchers about a new run\n assert mock_notify.call_args_list == [mock.call(job.Job.NOTIFY_NEW_RUN, event_data=new_run)]\n\n\nclass TestJobScheduler:\n @pytest.fixture(autouse=True)\n def setup_job(self):\n mock_graph = mock.Mock(autospec=True)\n mock_graph.get_action_map.return_value = {}\n mock_graph.action_map = {}\n self.job = mock.Mock(autospec=True)\n self.job.allow_overlap = False\n self.job.max_runtime = datetime.timedelta(days=1)\n self.job_scheduler = JobScheduler(job=self.job)\n\n def test_restore_state_sets_job_runs(self):\n self.job.enabled = False\n mock_runs = [mock.Mock(), mock.Mock()]\n mock_action_runner = mock.Mock()\n job_state_data = {\"runs\": mock_runs, \"enabled\": True}\n\n self.job_scheduler._set_callback = lambda x: x\n\n self.job.runs.runs = collections.deque()\n self.job.runs.get_scheduled.return_value = [mock.Mock()]\n self.job.get_job_runs_from_state.return_value = mock_runs\n\n with mock.patch(\n \"tron.core.job_scheduler.recovery.launch_recovery_actionruns_for_job_runs\",\n autospec=True,\n ) as mock_launch_recovery:\n mock_launch_recovery.return_value = mock.Mock(autospec=True)\n self.job_scheduler.restore_state(\n job_state_data,\n mock_action_runner,\n )\n assert self.job.runs.runs == collections.deque(mock_runs)\n mock_launch_recovery.assert_called_once_with(\n job_runs=mock_runs,\n master_action_runner=mock_action_runner,\n )\n calls = [mock.call(mock_runs[i]) for i in range(0, len(mock_runs))]\n self.job.watch.assert_has_calls(calls)\n\n def test_create_and_schedule_runs_specific_time(self):\n self.job_scheduler.get_runs_to_schedule = mock.Mock(return_value=[mock.Mock()])\n self.job_scheduler._set_callback = mock.Mock()\n self.job_scheduler.create_and_schedule_runs(next_run_time=\"a_datetime\")\n assert self.job_scheduler.get_runs_to_schedule.call_args_list == [mock.call(\"a_datetime\")]\n\n def test_create_and_schedule_runs_guess(self):\n self.job_scheduler.get_runs_to_schedule = mock.Mock(return_value=[mock.Mock()])\n self.job_scheduler._set_callback = mock.Mock()\n self.job_scheduler.create_and_schedule_runs(next_run_time=None)\n assert self.job_scheduler.get_runs_to_schedule.call_args_list == [mock.call(None)]\n\n def test_disable(self):\n self.job_scheduler.disable()\n assert self.job_scheduler.job.enabled is False\n self.job_scheduler.job.runs.cancel_pending.assert_called_once()\n\n def test_schedule_reconfigured(self):\n pending_run = mock.Mock()\n pending_run.run_time = \"a_run_time\"\n self.job.runs.get_pending.return_value = [pending_run]\n self.job_scheduler.create_and_schedule_runs = mock.Mock()\n\n self.job_scheduler.schedule_reconfigured()\n\n assert self.job.runs.remove_pending.call_count == 1\n assert self.job_scheduler.create_and_schedule_runs.call_args_list == [\n mock.call(\n next_run_time=\"a_run_time\",\n ),\n ]\n\n def test_schedule(self):\n self.job.enabled = True\n last_run = mock.Mock()\n last_run.run_time = \"a_run_time\"\n self.job.runs.get_newest = mock.Mock(return_value=last_run)\n self.job_scheduler.create_and_schedule_runs = mock.Mock()\n\n self.job_scheduler.schedule()\n\n self.job.scheduler.next_run_time.assert_called_once_with(\"a_run_time\")\n assert self.job_scheduler.create_and_schedule_runs.call_args_list == [\n mock.call(next_run_time=self.job.scheduler.next_run_time.return_value),\n ]\n\n def test_run_job(self):\n self.job_scheduler.schedule = mock.Mock(autospec=True)\n self.job.scheduler.schedule_on_complete = False\n self.job.runs.get_active = lambda n: []\n job_run = mock.Mock(autospec=True)\n job_run.is_cancelled = False\n self.job_scheduler.run_job(job_run)\n job_run.start.assert_called_once()\n self.job_scheduler.schedule.assert_called_once()\n\n def test_run_job_job_disabled(self):\n self.job_scheduler.schedule = MagicMock()\n job_run = MagicMock()\n self.job.enabled = False\n self.job_scheduler.run_job(job_run)\n assert_length(self.job_scheduler.schedule.mock_calls, 0)\n assert_length(job_run.start.mock_calls, 0)\n assert_length(job_run.cancel.mock_calls, 1)\n\n def test_run_job_cancelled(self):\n self.job_scheduler.schedule = MagicMock()\n job_run = MagicMock(is_scheduled=False)\n self.job_scheduler.run_job(job_run)\n assert_length(job_run.start.mock_calls, 0)\n assert_length(self.job_scheduler.schedule.mock_calls, 1)\n\n def test_run_job_already_running_queuing(self):\n self.job_scheduler.schedule = mock.Mock(autospec=True)\n self.job.runs.get_active = lambda s: [mock.Mock(autospec=True)]\n job_run = mock.Mock(autospec=True)\n job_run.is_cancelled = False\n self.job_scheduler.run_job(job_run)\n assert not job_run.start.called\n job_run.queue.assert_called_once()\n assert not self.job_scheduler.schedule.called\n\n def test_run_job_already_running_cancel(self):\n self.job_scheduler.schedule = mock.Mock(autospec=True)\n self.job.runs.get_active = lambda s: [mock.Mock(autospec=True)]\n self.job.queueing = False\n job_run = mock.Mock(autospec=True)\n job_run.is_cancelled = False\n self.job_scheduler.run_job(job_run)\n assert not job_run.start.called\n job_run.cancel.assert_called_once()\n self.job_scheduler.schedule.assert_called_once()\n\n def test_run_job_already_running_allow_overlap(self):\n self.job_scheduler.schedule = mock.Mock()\n self.job.runs.get_active = lambda s: [mock.Mock()]\n self.job.allow_overlap = True\n job_run = MagicMock(is_cancelled=False)\n self.job_scheduler.run_job(job_run)\n job_run.start.assert_called_with()\n\n def test_run_job_has_starting_queueing(self):\n self.job_scheduler.schedule = mock.Mock(autospec=True)\n self.job.runs.get_active = lambda s: [mock.Mock(autospec=True)]\n job_run = mock.Mock(autospec=True)\n job_run.is_cancelled = False\n self.job_scheduler.run_job(job_run)\n assert not job_run.start.called\n job_run.queue.assert_called_once()\n assert not self.job_scheduler.schedule.called\n\n def test_run_job_schedule_on_complete(self):\n self.job_scheduler.schedule = MagicMock()\n self.job.scheduler.schedule_on_complete = True\n self.job.runs.get_active = lambda s: []\n job_run = MagicMock(is_cancelled=False)\n self.job_scheduler.run_job(job_run)\n assert_length(job_run.start.mock_calls, 1)\n assert_length(self.job_scheduler.schedule.mock_calls, 0)\n" }, { "path": "tests/core/jobgraph_test.py", "content": "from unittest import mock\n\nimport pytest\n\nfrom tron.config.schema import ConfigAction\nfrom tron.config.schema import ConfigJob\nfrom tron.core.jobgraph import AdjListEntry\nfrom tron.core.jobgraph import JobGraph\n\n\nMISSING_DEPENDENCY_ERR_MSG = \"\"\"The following actions are dependencies of other actions but missing:\nAction other.job2.action3 is dependency of actions:\n - MASTER.job3.action5\nPlease check if you have deleted/renamed any of them or their containing jobs.\"\"\"\n\n\ndef _setup_job_graph_config_container():\n action1 = ConfigAction(\n name=\"action1\",\n command=\"do something\",\n )\n action2 = ConfigAction(\n name=\"action2\",\n command=\"do something\",\n requires=[\"action1\"],\n )\n job1_config = ConfigJob(\n name=\"job1\",\n node=\"default\",\n schedule=mock.Mock(),\n actions={\"action1\": action1, \"action2\": action2},\n namespace=\"MASTER\",\n )\n\n action3 = ConfigAction(\n name=\"action3\",\n command=\"do something\",\n triggered_by=[\"MASTER.job1.action2.shortdate.{shortdate}\"],\n )\n job2_config = ConfigJob(\n name=\"job1\",\n node=\"default\",\n schedule=mock.Mock(),\n actions={\"action3\": action3},\n namespace=\"other\",\n )\n\n action4 = ConfigAction(\n name=\"action4\",\n command=\"do something\",\n )\n action5 = ConfigAction(\n name=\"action5\",\n command=\"do something\",\n requires=[\"action4\"],\n triggered_by=[\"other.job2.action3.shortdate.{shortdate}\"],\n )\n job3_config = ConfigJob(\n name=\"job1\",\n node=\"default\",\n schedule=mock.Mock(),\n actions={\"action4\": action4, \"action5\": action5},\n namespace=\"MASTER\",\n )\n config_container = mock.Mock()\n config_container.get_jobs.return_value = {\n \"MASTER.job1\": job1_config,\n \"other.job2\": job2_config,\n \"MASTER.job3\": job3_config,\n }\n return config_container\n\n\nclass TestJobGraph:\n def setup_method(self):\n self.job_graph = JobGraph(_setup_job_graph_config_container(), should_validate_missing_dependency=True)\n\n def test_job_graph_missing_dependency(self):\n missing_dependency_config_container = _setup_job_graph_config_container()\n missing_dependency_config_container.get_jobs.return_value.pop(\"other.job2\")\n with pytest.raises(ValueError) as e:\n JobGraph(\n missing_dependency_config_container,\n should_validate_missing_dependency=True,\n )\n assert str(e.value) == MISSING_DEPENDENCY_ERR_MSG\n\n def test_job_graph(self):\n assert sorted(list(self.job_graph.action_map.keys())) == [\n \"MASTER.job1.action1\",\n \"MASTER.job1.action2\",\n \"MASTER.job3.action4\",\n \"MASTER.job3.action5\",\n \"other.job2.action3\",\n ]\n assert self.job_graph._actions_for_job == {\n \"MASTER.job1\": [\"MASTER.job1.action1\", \"MASTER.job1.action2\"],\n \"other.job2\": [\"other.job2.action3\"],\n \"MASTER.job3\": [\"MASTER.job3.action4\", \"MASTER.job3.action5\"],\n }\n assert self.job_graph._adj_list == {\n \"MASTER.job1.action1\": [AdjListEntry(\"MASTER.job1.action2\", False)],\n \"MASTER.job1.action2\": [AdjListEntry(\"other.job2.action3\", True)],\n \"other.job2.action3\": [AdjListEntry(\"MASTER.job3.action5\", True)],\n \"MASTER.job3.action4\": [AdjListEntry(\"MASTER.job3.action5\", False)],\n }\n assert self.job_graph._rev_adj_list == {\n \"MASTER.job1.action1\": [],\n \"MASTER.job1.action2\": [AdjListEntry(\"MASTER.job1.action1\", False)],\n \"other.job2.action3\": [AdjListEntry(\"MASTER.job1.action2\", True)],\n \"MASTER.job3.action4\": [],\n \"MASTER.job3.action5\": [\n AdjListEntry(\"MASTER.job3.action4\", False),\n AdjListEntry(\"other.job2.action3\", True),\n ],\n }\n\n def test_get_action_graph_for_job(self):\n action_graph_1 = self.job_graph.get_action_graph_for_job(\"MASTER.job1\")\n assert sorted(action_graph_1.action_map.keys()) == [\n \"action1\",\n \"action2\",\n ]\n assert action_graph_1.required_actions == {\n \"action1\": set(),\n \"action2\": {\"action1\"},\n }\n assert action_graph_1.required_triggers == {\n \"other.job2.action3\": {\"action2\"},\n \"MASTER.job3.action5\": {\"other.job2.action3\"},\n }\n\n action_graph_2 = self.job_graph.get_action_graph_for_job(\"other.job2\")\n assert sorted(action_graph_2.action_map.keys()) == [\n \"action3\",\n ]\n assert action_graph_2.required_actions == {\n \"action3\": set(),\n }\n assert action_graph_2.required_triggers == {\n \"action3\": {\"MASTER.job1.action2\"},\n \"MASTER.job3.action5\": {\"action3\"},\n }\n\n action_graph_3 = self.job_graph.get_action_graph_for_job(\"MASTER.job3\")\n assert sorted(action_graph_3.action_map.keys()) == [\n \"action4\",\n \"action5\",\n ]\n assert action_graph_3.required_actions == {\n \"action4\": set(),\n \"action5\": {\"action4\"},\n }\n assert action_graph_3.required_triggers == {\n \"action5\": {\"other.job2.action3\"},\n \"other.job2.action3\": {\"MASTER.job1.action2\"},\n }\n" }, { "path": "tests/core/jobrun_test.py", "content": "import datetime\nimport json\nfrom unittest import mock\nfrom unittest.mock import MagicMock\n\nimport pytest\nimport pytz\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_in\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tests.assertions import assert_call\nfrom tests.assertions import assert_length\nfrom tests.assertions import assert_raises\nfrom tests.testingutils import autospec_method\nfrom tron import actioncommand\nfrom tron import node\nfrom tron.core import action\nfrom tron.core import actiongraph\nfrom tron.core import actionrun\nfrom tron.core import job\nfrom tron.core import jobrun\nfrom tron.serialize import filehandler\n\n\ndef build_mock_job():\n action_graph = mock.create_autospec(actiongraph.ActionGraph)\n action_graph.action_map = {\n \"foo\": mock.Mock(\n triggered_by=[],\n trigger_timeout=datetime.timedelta(days=1),\n ),\n }\n runner = mock.create_autospec(actioncommand.SubprocessActionRunnerFactory)\n return mock.create_autospec(\n job.Job,\n action_graph=action_graph,\n output_path=mock.Mock(),\n context=mock.Mock(),\n action_runner=runner,\n )\n\n\nclass TestJobRun:\n\n now = datetime.datetime(2012, 3, 14, 15, 9, 20, tzinfo=None)\n now_with_tz = datetime.datetime(2012, 3, 14, 15, 9, 20, tzinfo=pytz.utc)\n\n @setup\n def setup_jobrun(self):\n self.job = build_mock_job()\n self.action_graph = self.job.action_graph\n self.run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)\n mock_node = mock.create_autospec(node.Node)\n self.job_run = jobrun.JobRun(\n \"jobname\",\n 7,\n self.run_time,\n mock_node,\n action_runs=MagicMock(\n action_runs_with_cleanup=[],\n get_startable_action_runs=lambda: [],\n ),\n )\n autospec_method(self.job_run.watch)\n autospec_method(self.job_run.notify)\n self.action_run = mock.create_autospec(\n actionrun.ActionRun,\n is_skipped=False,\n )\n\n def test__init__(self):\n assert_equal(self.job_run.job_name, \"jobname\")\n assert_equal(self.job_run.run_time, self.run_time)\n assert str(self.job_run.output_path).endswith(str(self.job_run.run_num))\n\n def test_for_job(self):\n run_num = 6\n mock_node = mock.create_autospec(node.Node)\n run = jobrun.JobRun.for_job(\n self.job,\n run_num,\n self.run_time,\n mock_node,\n False,\n )\n\n assert_equal(run.action_runs.action_graph, self.action_graph)\n assert_equal(run.job_name, self.job.get_name.return_value)\n assert_equal(run.run_num, run_num)\n assert_equal(run.node, mock_node)\n assert not run.manual\n\n def test_for_job_manual(self):\n run_num = 6\n mock_node = mock.create_autospec(node.Node)\n run = jobrun.JobRun.for_job(\n self.job,\n run_num,\n self.run_time,\n mock_node,\n True,\n )\n assert_equal(run.action_runs.action_graph, self.action_graph)\n assert run.manual\n\n def test_state_data(self):\n state_data = self.job_run.state_data\n assert_equal(state_data[\"run_num\"], 7)\n assert not state_data[\"manual\"]\n assert_equal(state_data[\"run_time\"], self.run_time)\n\n def test_set_action_runs(self):\n self.job_run._action_runs = None\n count = 2\n action_runs = [mock.create_autospec(actionrun.ActionRun) for _ in range(count)]\n run_collection = mock.create_autospec(\n actionrun.ActionRunCollection,\n action_runs_with_cleanup=action_runs,\n )\n self.job_run._set_action_runs(run_collection)\n assert_equal(self.job_run.watch.call_count, count)\n\n expected = [mock.call(run) for run in action_runs]\n assert_equal(self.job_run.watch.mock_calls, expected)\n assert_equal(self.job_run.action_runs, run_collection)\n assert self.job_run.action_runs_proxy\n\n def test_set_action_runs_none(self):\n self.job_run._action_runs = None\n run_collection = mock.create_autospec(actionrun.ActionRunCollection)\n self.job_run._set_action_runs(run_collection)\n assert not self.job_run.watch.mock_calls\n assert_equal(self.job_run.action_runs, run_collection)\n\n def test_set_action_runs_duplicate(self):\n run_collection = mock.create_autospec(actionrun.ActionRunCollection)\n assert_raises(\n ValueError,\n self.job_run._set_action_runs,\n run_collection,\n )\n\n @mock.patch(\"tron.core.jobrun.timeutils.current_time\", autospec=True)\n def test_seconds_until_run_time(self, mock_current_time):\n mock_current_time.return_value = self.now\n seconds = self.job_run.seconds_until_run_time()\n assert_equal(seconds, 6)\n\n @mock.patch(\"tron.core.jobrun.timeutils.current_time\", autospec=True)\n def test_seconds_until_run_time_with_tz(self, mock_current_time):\n mock_current_time.return_value = self.now_with_tz\n self.job_run.run_time = self.run_time.replace(tzinfo=pytz.utc)\n seconds = self.job_run.seconds_until_run_time()\n assert_equal(seconds, 6)\n\n def test_start(self):\n autospec_method(self.job_run._do_start)\n assert self.job_run.start()\n self.job_run._do_start.assert_called_with()\n\n def test_start_failed(self):\n autospec_method(self.job_run._do_start, return_value=False)\n assert not self.job_run.start()\n\n def test_do_start(self):\n startable_runs = [mock.create_autospec(actionrun.ActionRun) for _ in range(3)]\n self.job_run.action_runs.get_startable_action_runs = lambda: startable_runs\n\n assert self.job_run._do_start()\n self.job_run.action_runs.ready.assert_called_with()\n for startable_run in startable_runs:\n startable_run.start.assert_called_with()\n\n def test_do_start_all_failed(self):\n autospec_method(self.job_run._start_action_runs, return_value=[None])\n assert not self.job_run._do_start()\n\n def test_do_start_some_failed(self):\n returns = [True, None]\n autospec_method(self.job_run._start_action_runs, return_value=returns)\n assert self.job_run._do_start()\n\n def test_do_start_no_runs(self):\n assert not self.job_run._do_start()\n\n def test_start_action_runs(self):\n startable_runs = [mock.create_autospec(actionrun.ActionRun) for _ in range(3)]\n self.job_run.action_runs.get_startable_action_runs = lambda: startable_runs\n\n started_runs = self.job_run._start_action_runs()\n assert_equal(started_runs, startable_runs)\n\n def test_start_action_runs_failed(self):\n startable_runs = [mock.create_autospec(actionrun.ActionRun) for _ in range(3)]\n startable_runs[0].start.return_value = False\n self.job_run.action_runs.get_startable_action_runs = lambda: startable_runs\n\n started_runs = self.job_run._start_action_runs()\n assert_equal(started_runs, startable_runs[1:])\n\n @pytest.fixture\n def jobrun_json(self):\n runs = [\n {\n \"job_run_id\": \"compute-infra-test-service.test_load_foo1.5910\",\n \"action_name\": \"example_action\",\n \"state\": \"succeeded\",\n \"original_command\": \"date; sleep 150; date\",\n \"start_time\": \"2023-10-01T12:00:00\",\n \"end_time\": \"2023-10-01T12:30:00\",\n \"node_name\": \"paasta\",\n \"exit_status\": 0,\n \"attempts\": [],\n \"retries_remaining\": 2,\n \"retries_delay\": 60,\n \"action_runner\": '{\"status_path\": \"/tmp/tron\", \"exec_path\": \"/opt/venvs/tron/bin\"}',\n \"executor\": \"kubernetes\",\n \"trigger_timeout_timestamp\": 1731584100,\n \"trigger_downstreams\": False,\n \"triggered_by\": [],\n \"on_upstream_rerun\": None,\n }\n ]\n cleanup = {\n \"job_run_id\": \"compute-infra-test-service.test_load_foo1.5910\",\n \"action_name\": \"cleanup_action\",\n \"state\": \"succeeded\",\n \"original_command\": \"date; sleep 150; date\",\n \"start_time\": \"2023-10-01T12:00:00\",\n \"end_time\": \"2023-10-01T12:30:00\",\n \"node_name\": \"paasta\",\n \"exit_status\": 0,\n \"attempts\": [],\n \"retries_remaining\": 2,\n \"retries_delay\": 60,\n \"action_runner\": '{\"status_path\": \"/tmp/tron\", \"exec_path\": \"/opt/venvs/tron/bin\"}',\n \"executor\": \"kubernetes\",\n \"trigger_timeout_timestamp\": 1731584100,\n \"trigger_downstreams\": False,\n \"triggered_by\": [],\n \"on_upstream_rerun\": None,\n }\n serialized_cleanup = json.dumps(cleanup)\n serialized_runs = [json.dumps(run) for run in runs]\n return json.dumps(\n {\n \"job_name\": \"example_job\",\n \"run_num\": 1,\n \"run_time\": \"2023-10-01T12:00:00\",\n \"time_zone\": None,\n \"node_name\": \"example_node\",\n \"runs\": serialized_runs,\n \"cleanup_run\": serialized_cleanup,\n \"manual\": False,\n }\n )\n\n def test_from_json(self, jobrun_json):\n result = jobrun.JobRun.from_json(jobrun_json)\n expected = {\n \"job_name\": \"example_job\",\n \"run_num\": 1,\n \"run_time\": datetime.datetime(2023, 10, 1, 12, 0, 0),\n \"node_name\": \"example_node\",\n \"runs\": [\n {\n \"job_run_id\": \"compute-infra-test-service.test_load_foo1.5910\",\n \"action_name\": \"example_action\",\n \"state\": \"succeeded\",\n \"original_command\": \"date; sleep 150; date\",\n \"start_time\": datetime.datetime(2023, 10, 1, 12, 0, 0),\n \"end_time\": datetime.datetime(2023, 10, 1, 12, 30, 0),\n \"node_name\": \"paasta\",\n \"exit_status\": 0,\n \"attempts\": [],\n \"retries_remaining\": 2,\n \"retries_delay\": datetime.timedelta(seconds=60),\n \"action_runner\": {\"status_path\": \"/tmp/tron\", \"exec_path\": \"/opt/venvs/tron/bin\"},\n \"executor\": \"kubernetes\",\n \"trigger_timeout_timestamp\": 1731584100,\n \"trigger_downstreams\": False,\n \"triggered_by\": [],\n \"on_upstream_rerun\": None,\n }\n ],\n \"cleanup_run\": {\n \"job_run_id\": \"compute-infra-test-service.test_load_foo1.5910\",\n \"action_name\": \"cleanup_action\",\n \"state\": \"succeeded\",\n \"original_command\": \"date; sleep 150; date\",\n \"start_time\": datetime.datetime(2023, 10, 1, 12, 0, 0),\n \"end_time\": datetime.datetime(2023, 10, 1, 12, 30, 0),\n \"node_name\": \"paasta\",\n \"exit_status\": 0,\n \"attempts\": [],\n \"retries_remaining\": 2,\n \"retries_delay\": datetime.timedelta(seconds=60),\n \"action_runner\": {\"status_path\": \"/tmp/tron\", \"exec_path\": \"/opt/venvs/tron/bin\"},\n \"executor\": \"kubernetes\",\n \"trigger_timeout_timestamp\": 1731584100,\n \"trigger_downstreams\": False,\n \"triggered_by\": [],\n \"on_upstream_rerun\": None,\n },\n \"manual\": False,\n \"time_zone\": None,\n }\n assert result == expected\n\n def test_start_action_runs_all_failed(self):\n startable_runs = [mock.create_autospec(actionrun.ActionRun) for _ in range(2)]\n for startable_run in startable_runs:\n startable_run.start.return_value = False\n self.job_run.action_runs.get_startable_action_runs = lambda: startable_runs\n\n started_runs = self.job_run._start_action_runs()\n assert_equal(started_runs, [])\n\n def test_handler_trigger_ready_still_scheduled(self):\n autospec_method(self.job_run._start_action_runs)\n self.job_run.is_scheduled = True\n self.job_run.handler(self.action_run, actionrun.ActionRun.NOTIFY_TRIGGER_READY)\n assert not self.job_run._start_action_runs.mock_calls\n\n def test_handler_trigger_ready_started(self):\n autospec_method(self.job_run._start_action_runs)\n self.job_run.is_scheduled = False\n self.job_run.is_queued = False\n self.job_run.handler(self.action_run, actionrun.ActionRun.NOTIFY_TRIGGER_READY)\n assert self.job_run._start_action_runs.call_count == 1\n\n def test_handler_not_end_state_event(self):\n autospec_method(self.job_run.finalize)\n autospec_method(self.job_run._start_action_runs)\n self.action_run.is_done = False\n self.job_run.handler(self.action_run, mock.Mock())\n assert not self.job_run.finalize.mock_calls\n assert not self.job_run._start_action_runs.mock_calls\n\n def test_handler_with_startable(self):\n startable_run = mock.create_autospec(actionrun.ActionRun)\n self.job_run.action_runs.get_startable_action_runs = lambda: [\n startable_run,\n ]\n autospec_method(self.job_run.finalize)\n self.action_run.is_broken = False\n\n self.job_run.handler(self.action_run, mock.Mock())\n self.job_run.notify.assert_called_with(\n self.job_run.NOTIFY_STATE_CHANGED,\n )\n startable_run.start.assert_called_with()\n assert not self.job_run.finalize.mock_calls\n\n def test_handler_runs_not_done(self):\n self.job_run.action_runs.is_done = False\n autospec_method(self.job_run._start_action_runs, return_value=[])\n autospec_method(self.job_run.finalize)\n self.job_run.handler(self.action_run, mock.Mock())\n assert not self.job_run.finalize.mock_calls\n\n def test_handler_finished_without_cleanup(self):\n self.job_run.action_runs.is_active = False\n self.job_run.action_runs.is_scheduled = False\n self.job_run.action_runs.cleanup_action_run = None\n autospec_method(self.job_run.finalize)\n self.job_run.handler(self.action_run, mock.Mock())\n self.job_run.finalize.assert_called_with()\n\n def test_handler_finished_with_cleanup_done(self):\n self.job_run.action_runs.is_active = False\n self.job_run.action_runs.is_scheduled = False\n self.job_run.action_runs.cleanup_action_run = mock.Mock(is_done=True)\n autospec_method(self.job_run.finalize)\n self.job_run.handler(self.action_run, mock.Mock())\n self.job_run.finalize.assert_called_with()\n\n def test_handler_finished_with_cleanup(self):\n self.job_run.action_runs.is_active = False\n self.job_run.action_runs.is_scheduled = False\n self.job_run.action_runs.cleanup_action_run = mock.Mock(is_done=False)\n autospec_method(self.job_run.finalize)\n self.job_run.handler(self.action_run, mock.Mock())\n assert not self.job_run.finalize.mock_calls\n self.job_run.action_runs.cleanup_action_run.start.assert_called_with()\n\n def test_handler_action_run_cancelled(self):\n self.action_run.is_broken = True\n autospec_method(self.job_run._start_action_runs)\n self.job_run.handler(self.action_run, mock.Mock())\n assert not self.job_run._start_action_runs.mock_calls\n\n def test_handler_action_run_skipped(self):\n self.action_run.is_broken = False\n self.action_run.is_skipped = True\n self.job_run.action_runs.is_scheduled = True\n autospec_method(self.job_run._start_action_runs)\n self.job_run.handler(self.action_run, mock.Mock())\n assert not self.job_run._start_action_runs.mock_calls\n\n def test_state(self):\n assert_equal(self.job_run.state, actionrun.ActionRun.SUCCEEDED)\n\n def test_state_with_no_action_runs(self):\n self.job_run._action_runs = None\n assert_equal(self.job_run.state, actionrun.ActionRun.UNKNOWN)\n\n def test_finalize(self):\n self.job_run.action_runs.is_failed = False\n self.job_run.finalize()\n self.job_run.notify.assert_called_with(self.job_run.NOTIFY_DONE)\n\n def test_finalize_failure(self):\n self.job_run.finalize()\n self.job_run.notify.assert_called_with(self.job_run.NOTIFY_DONE)\n\n def test_cleanup(self):\n autospec_method(self.job_run.clear_observers)\n self.job_run.output_path = mock.create_autospec(filehandler.OutputPath)\n self.job_run.cleanup()\n\n self.job_run.notify.assert_called_with(jobrun.JobRun.NOTIFY_REMOVED)\n self.job_run.clear_observers.assert_called_with()\n self.job_run.output_path.delete.assert_called_with()\n assert not self.job_run.node\n assert not self.job_run.action_graph\n assert not self.job_run.action_runs\n\n def test__getattr__(self):\n assert self.job_run.cancel\n assert self.job_run.state == \"succeeded\"\n assert self.job_run.is_succeeded\n\n def test__getattr__miss(self):\n assert_raises(AttributeError, lambda: self.job_run.bogus)\n\n\nclass TestJobRunFromState(TestCase):\n @setup\n def setup_jobrun(self):\n self.action_graph = mock.create_autospec(actiongraph.ActionGraph, action_map={})\n self.run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)\n self.path = [\"base\", \"path\"]\n self.output_path = mock.create_autospec(filehandler.OutputPath)\n self.node_pool = mock.create_autospec(node.NodePool)\n self.action_run_state_data = [\n {\n \"job_run_id\": \"thejobname.22\",\n \"action_name\": \"blingaction\",\n \"state\": \"succeeded\",\n \"run_time\": \"sometime\",\n \"start_time\": \"sometime\",\n \"end_time\": \"sometime\",\n \"command\": \"doit\",\n \"node_name\": \"thenode\",\n }\n ]\n self.state_data = {\n \"job_name\": \"thejobname\",\n \"run_num\": 22,\n \"run_time\": self.run_time,\n \"node_name\": \"thebox\",\n \"end_time\": \"the_end\",\n \"start_time\": \"start_time\",\n \"runs\": self.action_run_state_data,\n \"cleanup_run\": None,\n \"manual\": True,\n }\n self.context = mock.Mock()\n\n def test_from_state(self):\n run = jobrun.JobRun.from_state(\n self.state_data,\n self.action_graph,\n self.output_path,\n self.context,\n self.node_pool,\n )\n assert_length(run.action_runs.run_map, 1)\n assert_equal(run.job_name, self.state_data[\"job_name\"])\n assert_equal(run.run_time, self.run_time)\n assert run.manual\n assert_equal(run.output_path, self.output_path)\n assert run.context.next\n assert run.action_graph\n\n def test_from_state_node_no_longer_exists(self):\n run = jobrun.JobRun.from_state(\n self.state_data,\n self.action_graph,\n self.output_path,\n self.context,\n self.node_pool,\n )\n assert_length(run.action_runs.run_map, 1)\n assert_equal(run.job_name, \"thejobname\")\n assert_equal(run.run_time, self.run_time)\n assert_equal(run.node, self.node_pool)\n\n\nclass MockJobRun(MagicMock):\n\n manual = False\n\n node = \"anode\"\n\n @property\n def is_scheduled(self):\n return self.state == actionrun.ActionRun.SCHEDULED\n\n @property\n def is_queued(self):\n return self.state == actionrun.ActionRun.QUEUED\n\n @property\n def is_running(self):\n return self.state == actionrun.ActionRun.RUNNING\n\n @property\n def is_starting(self):\n return self.state == actionrun.ActionRun.STARTING\n\n @property\n def is_waiting(self):\n return self.state == actionrun.ActionRun.WAITING\n\n def __repr__(self):\n return str(self.__dict__)\n\n\nclass TestJobRunCollection(TestCase):\n def _mock_run(self, **kwargs):\n return MockJobRun(**kwargs)\n\n @setup\n def setup_runs(self):\n self.run_collection = jobrun.JobRunCollection(6)\n self.job_runs = [\n self._mock_run(state=actionrun.ActionRun.QUEUED, run_num=5),\n self._mock_run(state=actionrun.ActionRun.WAITING, run_num=4),\n self._mock_run(state=actionrun.ActionRun.RUNNING, run_num=3),\n ] + [\n self._mock_run(\n state=actionrun.ActionRun.SUCCEEDED,\n run_num=i,\n )\n for i in range(2, 0, -1)\n ]\n self.run_collection.runs.extend(self.job_runs)\n self.mock_node = mock.create_autospec(node.Node)\n\n def test__init__(self):\n assert_equal(self.run_collection.run_limit, 6)\n\n def test_from_config(self):\n job_config = mock.Mock(run_limit=20)\n runs = jobrun.JobRunCollection.from_config(job_config)\n assert_equal(runs.run_limit, 20)\n\n def test_job_runs_from_state(self):\n state_data = [\n dict(\n run_num=i,\n job_name=\"thename\",\n run_time=\"sometime\",\n start_time=\"start_time\",\n end_time=\"sometime\",\n cleanup_run=None,\n runs=[],\n )\n for i in range(3, -1, -1)\n ]\n action_graph = mock.create_autospec(actiongraph.ActionGraph)\n output_path = mock.create_autospec(filehandler.OutputPath)\n context = mock.Mock()\n node_pool = mock.create_autospec(node.NodePool)\n runs = jobrun.job_runs_from_state(\n state_data,\n action_graph,\n output_path,\n context,\n node_pool,\n )\n assert len(runs) == 4\n assert all([type(job) == jobrun.JobRun for job in runs])\n\n def test_build_new_run(self):\n autospec_method(self.run_collection.remove_old_runs)\n run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)\n mock_job = build_mock_job()\n job_run = self.run_collection.build_new_run(\n mock_job,\n run_time,\n self.mock_node,\n )\n assert_in(job_run, self.run_collection.runs)\n self.run_collection.remove_old_runs.assert_called_with()\n assert job_run.run_num == 6\n assert job_run.job_name == mock_job.get_name.return_value\n\n def test_build_new_run_manual(self):\n autospec_method(self.run_collection.remove_old_runs)\n run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)\n mock_job = build_mock_job()\n job_run = self.run_collection.build_new_run(\n mock_job,\n run_time,\n self.mock_node,\n True,\n )\n assert_in(job_run, self.run_collection.runs)\n self.run_collection.remove_old_runs.assert_called_with()\n assert job_run.run_num == 6\n assert job_run.manual\n\n def test_cancel_pending(self):\n pending_runs = [mock.Mock() for _ in range(2)]\n autospec_method(\n self.run_collection.get_pending,\n return_value=pending_runs,\n )\n self.run_collection.cancel_pending()\n for pending_run in pending_runs:\n pending_run.cancel.assert_called_with()\n\n def test_cancel_pending_no_pending(self):\n autospec_method(self.run_collection.get_pending, return_value=[])\n self.run_collection.cancel_pending()\n\n def test_remove_pending(self):\n self.run_collection.remove_pending()\n assert_length(self.run_collection.runs, 4)\n assert_equal(self.run_collection.runs[0], self.job_runs[1])\n assert_call(self.job_runs[0].cleanup, 0)\n\n def test_get_run_by_state(self):\n state = actionrun.ActionRun.SUCCEEDED\n run = self.run_collection.get_run_by_state(state)\n assert_equal(run, self.job_runs[3])\n\n def test_get_run_by_state_no_match(self):\n state = actionrun.ActionRun.UNKNOWN\n run = self.run_collection.get_run_by_state(state)\n assert_equal(run, None)\n\n def test_get_run_by_num(self):\n run = self.run_collection.get_run_by_num(1)\n assert_equal(run.run_num, 1)\n\n def test_get_run_by_num_no_match(self):\n run = self.run_collection.get_run_by_num(7)\n assert_equal(run, None)\n\n def test_get_run_by_index(self):\n run = self.run_collection.get_run_by_index(-1)\n assert_equal(run, self.job_runs[0])\n run = self.run_collection.get_run_by_index(-2)\n assert_equal(run, self.job_runs[1])\n run = self.run_collection.get_run_by_index(0)\n assert_equal(run, self.job_runs[-1])\n run = self.run_collection.get_run_by_index(1)\n assert_equal(run, self.job_runs[-2])\n\n def test_get_run_by_index_invalid_index(self):\n run = self.run_collection.get_run_by_index(-6)\n assert_equal(run, None)\n run = self.run_collection.get_run_by_index(5)\n assert_equal(run, None)\n\n def test_get_newest(self):\n run = self.run_collection.get_newest()\n assert_equal(run, self.job_runs[0])\n\n def test_get_newest_exclude_manual(self):\n run = self._mock_run(\n state=actionrun.ActionRun.RUNNING,\n run_num=5,\n manual=True,\n )\n self.job_runs.insert(0, run)\n newest_run = self.run_collection.get_newest(include_manual=False)\n assert_equal(newest_run, self.job_runs[1])\n\n def test_get_newest_no_runs(self):\n run_collection = jobrun.JobRunCollection(5)\n assert_equal(run_collection.get_newest(), None)\n\n def test_pending(self):\n run_num = self.run_collection.next_run_num()\n scheduled_run = self._mock_run(\n run_num=run_num,\n state=actionrun.ActionRun.SCHEDULED,\n )\n self.run_collection.runs.appendleft(scheduled_run)\n pending = list(self.run_collection.get_pending())\n assert_length(pending, 2)\n assert_equal(pending, [scheduled_run, self.job_runs[0]])\n\n def test_get_active(self):\n starting_run = self._mock_run(\n run_num=self.run_collection.next_run_num(),\n state=actionrun.ActionRun.STARTING,\n )\n self.run_collection.runs.appendleft(starting_run)\n active = list(self.run_collection.get_active())\n assert_length(active, 3)\n assert_equal(active, [starting_run, self.job_runs[1], self.job_runs[2]])\n\n def test_get_active_with_node(self):\n starting_run = self._mock_run(\n run_num=self.run_collection.next_run_num(),\n state=actionrun.ActionRun.STARTING,\n )\n starting_run.node = \"differentnode\"\n self.run_collection.runs.appendleft(starting_run)\n active = list(self.run_collection.get_active(\"anode\"))\n assert_length(active, 2)\n assert_equal(active, [self.job_runs[1], self.job_runs[2]])\n\n def test_get_active_none(self):\n active = list(self.run_collection.get_active(\"bogus\"))\n assert_length(active, 0)\n\n def test_get_first_queued(self):\n run_num = self.run_collection.next_run_num()\n second_queued = self._mock_run(\n run_num=run_num,\n state=actionrun.ActionRun.QUEUED,\n )\n self.run_collection.runs.appendleft(second_queued)\n\n first_queued = self.run_collection.get_first_queued()\n assert_equal(first_queued, self.job_runs[0])\n\n def test_get_first_queued_no_match(self):\n self.job_runs[0].state = actionrun.ActionRun.CANCELLED\n first_queued = self.run_collection.get_first_queued()\n assert not first_queued\n\n def test_get_next_run_num(self):\n assert_equal(self.run_collection.next_run_num(), 6)\n\n def test_get_next_run_num_first(self):\n run_collection = jobrun.JobRunCollection(5)\n assert_equal(run_collection.next_run_num(), 0)\n\n def test_remove_old_runs(self):\n self.run_collection.run_limit = 1\n self.run_collection.remove_old_runs()\n\n assert_length(self.run_collection.runs, 1)\n assert_call(self.job_runs[-1].cleanup, 0)\n for job_run in self.run_collection.runs:\n assert_length(job_run.cancel.calls, 0)\n\n def test_remove_old_runs_none(self):\n self.run_collection.remove_old_runs()\n for job_run in self.job_runs:\n assert_length(job_run.cancel.calls, 0)\n\n def test_remove_old_runs_no_runs(self):\n run_collection = jobrun.JobRunCollection(4)\n run_collection.remove_old_runs()\n\n def test_state_data(self):\n assert_length(self.run_collection.state_data, len(self.job_runs))\n\n def test_last_success(self):\n assert_equal(self.run_collection.last_success, self.job_runs[3])\n\n def test__str__(self):\n expected = \"JobRunCollection[5(queued), 4(waiting), 3(running), 2(succeeded), 1(succeeded)]\"\n assert_equal(str(self.run_collection), expected)\n\n def test_get_action_runs(self):\n action_name = \"action_name\"\n self.run_collection.runs = job_runs = [mock.Mock(), mock.Mock()]\n runs = self.run_collection.get_action_runs(action_name)\n expected = [job_run.get_action_run.return_value for job_run in job_runs]\n assert_equal(runs, expected)\n for job_run in job_runs:\n job_run.get_action_run.assert_called_with(action_name)\n\n def test_get_run_nums(self):\n assert self.run_collection.get_run_nums() == [5, 4, 3, 2, 1]\n\n\nclass TestJobRunStateTransitions:\n \"\"\"Integration test for the state of a job run when actions change state in various ways.\"\"\"\n\n @pytest.fixture\n def mock_event_bus(self):\n with mock.patch(\n \"tron.core.actionrun.EventBus\",\n autospec=True,\n ) as mock_event_bus:\n mock_event_bus.has_event.return_value = True\n yield mock_event_bus\n\n @pytest.fixture\n def job_run(self, tmpdir, mock_event_bus):\n action_foo = action.Action(\"foo\", action.ActionCommandConfig(\"command\"), None)\n action_after_foo = action.Action(\"after_foo\", action.ActionCommandConfig(\"command\"), None)\n action_bar = action.Action(\"bar\", action.ActionCommandConfig(\"command\"), None, triggered_by={\"trigger\"})\n action_graph = actiongraph.ActionGraph(\n action_map={\n \"foo\": action_foo,\n \"after_foo\": action_after_foo,\n \"bar\": action_bar,\n },\n required_actions={\"foo\": set(), \"after_foo\": {\"foo\"}, \"bar\": set()},\n required_triggers={\"foo\": set(), \"after_foo\": set(), \"bar\": {\"trigger\"}},\n )\n mock_job = mock.Mock(\n output_path=filehandler.OutputPath(tmpdir),\n action_graph=action_graph,\n action_runner=actioncommand.NoActionRunnerFactory(),\n )\n job_run = jobrun.JobRun.for_job(\n mock_job,\n run_num=1,\n run_time=datetime.datetime.now(),\n node=mock.Mock(),\n manual=False,\n )\n return job_run\n\n def test_success_path(self, job_run):\n # Check expected states as actions run normally and succeed.\n foo = job_run.get_action_run(\"foo\")\n after_foo = job_run.get_action_run(\"after_foo\")\n bar = job_run.get_action_run(\"bar\")\n\n # Run is initially SCHEDULED\n assert job_run.state == actionrun.ActionRun.SCHEDULED\n\n # After starting, both actions without dependencies start.\n # Run is STARTING\n job_run.start()\n assert foo.is_starting\n assert bar.is_starting\n assert job_run.state == actionrun.ActionRun.STARTING\n\n # Commands start successfully, run is RUNNING.\n foo.action_command.started()\n bar.action_command.started()\n assert job_run.state == actionrun.ActionRun.RUNNING\n\n # Still RUNNING after one of two running actions succeeds\n bar.action_command.exited(0)\n assert job_run.state == actionrun.ActionRun.RUNNING\n\n # after_foo starts after its dependency succeeds\n foo.action_command.exited(0)\n assert after_foo.is_starting\n after_foo.action_command.started()\n assert job_run.state == actionrun.ActionRun.RUNNING\n\n # SUCCEEDED after all actions succeed\n after_foo.action_command.exited(0)\n assert job_run.state == actionrun.ActionRun.SUCCEEDED\n\n def test_one_action_fails(self, job_run):\n foo = job_run.get_action_run(\"foo\")\n after_foo = job_run.get_action_run(\"after_foo\")\n bar = job_run.get_action_run(\"bar\")\n\n # bar action fails, job is RUNNING because foo is still running\n job_run.start()\n foo.action_command.started()\n bar.action_command.started()\n bar.action_command.exited(1)\n assert job_run.state == actionrun.ActionRun.RUNNING\n\n # after_foo still starts after its dependency succeeds\n foo.action_command.exited(0)\n assert after_foo.is_starting\n after_foo.action_command.started()\n assert job_run.state == actionrun.ActionRun.RUNNING\n\n # After running actions finish, run enters FAILED terminal state\n after_foo.action_command.exited(0)\n assert job_run.state == actionrun.ActionRun.FAILED\n\n # If we skip the failed action, run becomes SUCCEEDED\n bar.skip()\n assert job_run.state == actionrun.ActionRun.SUCCEEDED\n\n def test_one_action_unknown(self, job_run):\n foo = job_run.get_action_run(\"foo\")\n after_foo = job_run.get_action_run(\"after_foo\")\n bar = job_run.get_action_run(\"bar\")\n\n assert job_run.state == actionrun.ActionRun.SCHEDULED\n\n # bar action becomes unknown, job is RUNNING because foo is still running\n job_run.start()\n foo.action_command.started()\n bar.action_command.started()\n bar.action_command.exited(None)\n assert job_run.state == actionrun.ActionRun.RUNNING\n\n # after_foo still starts after its dependency succeeds\n foo.action_command.exited(0)\n assert after_foo.is_starting\n after_foo.action_command.started()\n assert job_run.state == actionrun.ActionRun.RUNNING\n\n # UNKNOWN after running actions finish\n after_foo.action_command.exited(0)\n assert job_run.state == actionrun.ActionRun.UNKNOWN\n\n def test_both_unknown_and_failed(self, job_run):\n foo = job_run.get_action_run(\"foo\")\n after_foo = job_run.get_action_run(\"after_foo\")\n bar = job_run.get_action_run(\"bar\")\n\n # bar action becomes unknown, job is RUNNING because foo is still running\n job_run.start()\n foo.action_command.started()\n bar.action_command.started()\n bar.action_command.exited(None)\n assert job_run.state == actionrun.ActionRun.RUNNING\n\n # after_foo still starts after its dependency succeeds\n foo.action_command.exited(0)\n assert after_foo.is_starting\n after_foo.action_command.started()\n assert job_run.state == actionrun.ActionRun.RUNNING\n\n # A different action fails\n # Overall run is FAILED\n after_foo.action_command.exited(1)\n assert job_run.state == actionrun.ActionRun.FAILED\n\n def test_required_action_fails(self, job_run):\n foo = job_run.get_action_run(\"foo\")\n after_foo = job_run.get_action_run(\"after_foo\")\n bar = job_run.get_action_run(\"bar\")\n\n assert job_run.state == actionrun.ActionRun.SCHEDULED\n\n # An action (foo) required by another action fails\n # Run is RUNNING while the other action, bar, is running\n job_run.start()\n foo.action_command.started()\n bar.action_command.started()\n foo.action_command.exited(1)\n assert job_run.state == actionrun.ActionRun.RUNNING\n\n # bar action succeeds\n # after_foo cannot run because its required action failed\n # So run is FAILED even though after_foo is waiting\n bar.action_command.exited(0)\n assert after_foo.is_waiting\n assert job_run.state == actionrun.ActionRun.FAILED\n\n # Pretend we reconfigured and after_foo doesn't depend on foo anymore\n # Run should not be WAITING\n # Ideally it would still be FAILED, but for now it's UNKNOWN in this case\n job_run.action_runs.action_graph.required_actions[\"after_foo\"] = {}\n assert job_run.state == actionrun.ActionRun.UNKNOWN\n\n def test_required_action_unknown(self, job_run):\n foo = job_run.get_action_run(\"foo\")\n after_foo = job_run.get_action_run(\"after_foo\")\n bar = job_run.get_action_run(\"bar\")\n\n # An action (foo) required by another action becomes unknown\n # Run is RUNNING while the other action, bar, is running\n job_run.start()\n foo.action_command.started()\n bar.action_command.started()\n foo.action_command.exited(None)\n assert job_run.state == actionrun.ActionRun.RUNNING\n\n # Other action succeeds\n # after_foo cannot run because its required action is unknown\n # So run is UNKNOWN even though after_foo is waiting\n bar.action_command.exited(0)\n assert after_foo.is_waiting\n assert job_run.state == actionrun.ActionRun.UNKNOWN\n\n # Pretend we reconfigured and after_foo doesn't depend on foo anymore\n # Run should not be waiting\n job_run.action_runs.action_graph.required_actions[\"after_foo\"] = {}\n assert job_run.state == actionrun.ActionRun.UNKNOWN\n\n def test_with_trigger(self, job_run, mock_event_bus):\n foo = job_run.get_action_run(\"foo\")\n after_foo = job_run.get_action_run(\"after_foo\")\n bar = job_run.get_action_run(\"bar\")\n\n # Start without trigger for bar\n mock_event_bus.has_event.return_value = False\n # Job should still start in scheduled state\n assert job_run.state == actionrun.ActionRun.SCHEDULED\n\n # Only foo is able to start\n job_run.start()\n assert foo.is_starting\n assert bar.is_waiting\n assert job_run.state == actionrun.ActionRun.STARTING\n\n foo.action_command.started()\n assert job_run.state == actionrun.ActionRun.RUNNING\n\n # after_foo runs normally after foo succeeds\n foo.action_command.exited(0)\n assert after_foo.is_starting\n after_foo.action_command.started()\n assert job_run.state == actionrun.ActionRun.RUNNING\n\n # After after_foo succeeds, run is not done\n # WAITING because bar is still waiting for a trigger\n after_foo.action_command.exited(0)\n assert job_run.state == actionrun.ActionRun.WAITING\n\n # After trigger is available, job run finishes as normal\n mock_event_bus.has_event.return_value = True\n bar.trigger_notify()\n assert bar.is_starting\n bar.action_command.started()\n bar.action_command.exited(0)\n assert job_run.state == actionrun.ActionRun.SUCCEEDED\n\n def test_queued(self, job_run):\n assert job_run.state == actionrun.ActionRun.SCHEDULED\n job_run.queue()\n assert job_run.state == actionrun.ActionRun.QUEUED\n job_run.start()\n assert job_run.state == actionrun.ActionRun.STARTING\n\n def test_cancel_one(self, job_run):\n assert job_run.state == actionrun.ActionRun.SCHEDULED\n job_run.start()\n assert job_run.state == actionrun.ActionRun.STARTING\n job_run.get_action_run(\"after_foo\").cancel()\n assert job_run.state == actionrun.ActionRun.CANCELLED\n" }, { "path": "tests/core/recovery_test.py", "content": "from unittest import mock\nfrom unittest.mock import Mock\n\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tron.actioncommand import NoActionRunnerFactory\nfrom tron.actioncommand import SubprocessActionRunnerFactory\nfrom tron.core.actionrun import ActionRun\nfrom tron.core.actionrun import KubernetesActionRun\nfrom tron.core.actionrun import MesosActionRun\nfrom tron.core.actionrun import SSHActionRun\nfrom tron.core.recovery import filter_action_runs_needing_recovery\nfrom tron.core.recovery import launch_recovery_actionruns_for_job_runs\nfrom tron.utils import timeutils\n\n\nclass TestRecovery(TestCase):\n @setup\n def fake_action_runs(self):\n mock_unknown_machine = Mock(autospec=True)\n mock_ok_machine = Mock(autospec=True)\n\n mock_unknown_machine.state = ActionRun.UNKNOWN\n mock_ok_machine.state = ActionRun.SUCCEEDED\n self.action_runs = [\n SSHActionRun(\n job_run_id=\"test.unknown\",\n name=\"test.unknown\",\n node=Mock(),\n command_config=Mock(),\n machine=mock_unknown_machine,\n end_time=timeutils.current_time(),\n ),\n SSHActionRun(\n job_run_id=\"test.succeeded\",\n name=\"test.succeeded\",\n node=Mock(),\n command_config=Mock(),\n machine=mock_ok_machine,\n ),\n MesosActionRun(\n job_run_id=\"test.succeeded\",\n name=\"test.succeeded\",\n node=Mock(),\n command_config=Mock(),\n machine=mock_ok_machine,\n ),\n MesosActionRun(\n job_run_id=\"test.unknown-mesos\",\n name=\"test.unknown-mesos\",\n node=Mock(),\n command_config=Mock(),\n machine=mock_unknown_machine,\n ),\n MesosActionRun(\n job_run_id=\"test.unknown-mesos-done\",\n name=\"test.unknown-mesos-done\",\n node=Mock(),\n command_config=Mock(),\n machine=mock_unknown_machine,\n end_time=timeutils.current_time(),\n ),\n # TODO: Convert to all KubernetesActionRuns after deprecating mesos\n # A job will normally only ever have MesosActionRuns or KubernetsActionRuns\n KubernetesActionRun(\n job_run_id=\"test.k8s-done\",\n name=\"test.k8s-done\",\n node=Mock(),\n command_config=Mock(),\n machine=mock_unknown_machine,\n end_time=timeutils.current_time(),\n ),\n KubernetesActionRun(\n job_run_id=\"test.k8s-unknown\",\n name=\"test.k8s-unknown\",\n node=Mock(),\n command_config=Mock(),\n machine=mock_unknown_machine,\n ),\n ]\n\n def test_filter_action_runs_needing_recovery(self):\n assert filter_action_runs_needing_recovery(self.action_runs) == (\n [self.action_runs[0]],\n [self.action_runs[3]],\n [self.action_runs[6]],\n )\n\n @mock.patch(\"tron.core.recovery.filter_action_runs_needing_recovery\", autospec=True)\n def test_launch_recovery_actionruns_for_job_runs(self, mock_filter):\n mock_actions = (\n [\n mock.Mock(\n action_runner=NoActionRunnerFactory(),\n spec=SSHActionRun,\n ),\n mock.Mock(\n action_runner=SubprocessActionRunnerFactory(\n status_path=\"/tmp/foo\",\n exec_path=(\"/tmp/foo\"),\n ),\n spec=SSHActionRun,\n ),\n ],\n [\n mock.Mock(\n action_runner=NoActionRunnerFactory(),\n spec=MesosActionRun,\n ),\n ],\n [\n mock.Mock(\n action_runner=NoActionRunnerFactory(),\n spec=KubernetesActionRun,\n ),\n ],\n )\n\n mock_filter.return_value = mock_actions\n mock_action_runner = mock.Mock(autospec=True)\n\n mock_job_run = mock.Mock()\n launch_recovery_actionruns_for_job_runs(\n [mock_job_run],\n mock_action_runner,\n )\n ssh_runs = mock_actions[0]\n for run in ssh_runs:\n assert run.recover.call_count == 1\n\n mesos_run = mock_actions[1][0]\n assert mesos_run.recover.call_count == 1\n\n kubernetes_run = mock_actions[2][0]\n assert kubernetes_run.recover.call_count == 1\n\n @mock.patch(\"tron.core.recovery.filter_action_runs_needing_recovery\", autospec=True)\n def test_launch_recovery_actionruns_empty_job_run(self, mock_filter):\n \"\"\"_action_runs=None shouldn't prevent other job runs from being recovered\"\"\"\n empty_job_run = mock.Mock(_action_runs=None)\n other_job_run = mock.Mock(_action_runs=[mock.Mock()])\n mock_action_runner = mock.Mock()\n mock_filter.return_value = ([], [], [])\n\n launch_recovery_actionruns_for_job_runs(\n [empty_job_run, other_job_run],\n mock_action_runner,\n )\n mock_filter.assert_called_with(other_job_run._action_runs)\n" }, { "path": "tests/data/logging.conf", "content": "[loggers]\nkeys=root, twisted, tron\n\n[handlers]\nkeys=fileHandler\n\n[formatters]\nkeys=defaultFormatter\n\n[logger_root]\nlevel=WARN\nhandlers=fileHandler\n\n[logger_twisted]\nlevel=WARN\nhandlers=fileHandler\nqualname=twisted\npropagate=0\n\n[logger_tron]\nlevel=WARN\nhandlers=fileHandler\nqualname=tron\npropagate=0\n\n[handler_fileHandler]\nclass=logging.FileHandler\nlevel=WARN\nformatter=defaultFormatter\nargs=('{0}',)\n\n[formatter_defaultFormatter]\nformat=%(asctime)s %(name)s %(levelname)s %(message)s\n" }, { "path": "tests/data/test_config.yaml", "content": "# This test config is intended to cover most common configuration cases\n\nssh_options:\n agent: true\n\nstate_persistence:\n store_type: shelve\n name: tron_state.shelve\n buffer_size: 1\n\nnodes:\n - hostname: localhost\n - name: box1\n hostname: localhost\n - name: box2\n hostname: localhost\n - name: box3\n hostname: localhost\n - name: box4\n hostname: 127.0.0.1\n\nnode_pools:\n - name: pool0\n nodes: [localhost, box2]\n - name: pool1\n nodes: [box1]\n - nodes: [box1, box2]\n - name: pool2\n nodes: [box3, box4]\n\ncommand_context:\n THE_JOB_DIR: \"testconfig/jobs\"\n ECHO: \"echo\"\n # Change this to repo root\n PYTHON: \"cd /home/user/code/Tron && PYTHONPATH=. python\"\n\n\njobs:\n # IntervalScheduler no dependent Actions, single node\n - name: interval_job0\n run_limit: 3\n node: localhost\n schedule: \"cron * * * * *\"\n actions:\n - name: \"task0\"\n command: \"%(ECHO)s %(actionname)s\"\n - name: \"task1\"\n command: \"sleep 10 && %(ECHO)s %(actionname)s\"\n\n\n # IntervalScheduler dependent successful Actions, node pool\n - name: interval_job1\n node: pool0\n schedule: \"cron * * * * *\"\n actions:\n - name: task1\n command: \"%(ECHO)s %(actionname)s\"\n requires: [task0]\n - name: task0\n command: \"sleep 3 && %(ECHO)s %(actionname)s %(last_success:shortdate)s\"\n\n\n # IntervalScheduler dependent failure Actions\n - name: interval_job2\n node: box1_box2\n schedule: \"cron * * * * *\"\n actions:\n - name: task1\n command: \"%(ECHO)s %(actionname)s\"\n requires: [task0]\n - name: task0\n command: \"%(ECHO)s %(actionname)s && sleep 7 && false\"\n\n # Multiple dependent failure Actions\n - name: interval_job3\n node: box1_box2\n schedule: \"cron * * * * *\"\n actions:\n - name: task0\n command: \"%(ECHO)s %(actionname)s && sleep 7 && false\"\n - name: task1\n command: \"%(ECHO)s %(actionname)s\"\n requires: [task0]\n - name: task2\n command: \"%(ECHO)s %(actionname)s && sleep 10\"\n - name: task3\n command: \"%(ECHO)s %(actionname)s\"\n requires: [task2]\n\n\n # Multiple dependent failure Actions with cleanup\n - name: interval_job4\n node: box2\n schedule: \"cron * * * * *\"\n actions:\n - name: task0\n command: \"%(ECHO)s %(actionname)s && sleep 7 && false\"\n - name: task1\n command: \"%(ECHO)s %(actionname)s\"\n requires: [task0]\n - name: task2\n command: \"%(ECHO)s %(actionname)s && sleep 10\"\n - name: task3\n command: \"%(ECHO)s %(actionname)s && sleep 3\"\n requires: [task2]\n cleanup_action:\n command: \"%(ECHO)s %(actionname)s %(cleanup_job_status)s\"\n\n\n # No failures, with cleanup, different node for action\n - name: interval_job5\n node: box1\n schedule: \"cron * * * * *\"\n actions:\n - name: \"task0\"\n command: \"%(ECHO)s %(actionname)s\"\n node: box2\n - name: \"task1\"\n command: \"sleep 10 && %(ECHO)s %(actionname)s\"\n node: pool0\n cleanup_action:\n command: \"%(ECHO)s %(actionname)s %(cleanup_job_status)s\"\n\n\n # all_nodes Job\n - name: allnodes_job8\n node: pool2\n schedule: \"cron * * * * *\"\n all_nodes: true\n actions:\n - name: \"task0\"\n command: \"%(ECHO)s %(actionname)s\"\n - name: \"task1\"\n command: \"sleep 10 && %(ECHO)s %(actionname)s\"\n\n\n # Job failing bad action\n\n # DailyScheduler\n - name: daily_job9\n node: box1\n schedule: \"daily 16:00:00\"\n actions:\n - name: \"task0\"\n command: \"%(ECHO)s %(actionname)s 1 && false\"\n node: box2\n requires: [\"task1\"]\n - name: \"task1\"\n command: \"sleep 10 && %(ECHO)s %(actionname)s %(last_success:shortdate)s\"\n node: pool0\n cleanup_action:\n command: \"%(ECHO)s %(actionname)s %(cleanup_job_status)s\"\n\n # Overlapping, queueing\n - name: overlap_cancel\n node: pool2\n schedule: \"cron * * * * *\"\n queueing: false\n actions:\n - name: \"task0\"\n command: \"sleep 30s && %(ECHO)s %(actionname)s\"\n" }, { "path": "tests/eventbus_test.py", "content": "import os\nimport tempfile\nfrom collections import defaultdict\nfrom unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import setup\nfrom testifycompat import teardown\nfrom testifycompat import TestCase\nfrom tron.eventbus import EventBus\n\n\nclass MakeEventBusTestCase(TestCase):\n @setup\n def setup(self):\n self.logdir = tempfile.TemporaryDirectory()\n\n @teardown\n def teardown(self):\n EventBus.shutdown()\n self.logdir.cleanup()\n\n @mock.patch(\"tron.eventbus.time\", autospec=True)\n def test_setup_eventbus_dir(self, time):\n os.rmdir(self.logdir.name)\n\n time.time = mock.Mock(return_value=1.0)\n eb = EventBus.create(self.logdir.name)\n assert os.path.exists(self.logdir.name)\n assert os.path.exists(os.path.join(self.logdir.name, \"current\"))\n\n time.time = mock.Mock(return_value=2.0)\n eb.event_log = {\"foo\": \"bar\"}\n eb.sync_save_log(\"test\")\n\n new_eb = EventBus.create(self.logdir.name)\n new_eb.sync_load_log()\n assert new_eb.event_log == eb.event_log\n\n\nclass EventBusTestCase(TestCase):\n @setup\n def setup(self):\n self.log_dir = tempfile.TemporaryDirectory(prefix=\"tron_eventbus_test\")\n self.eventbus = EventBus.create(self.log_dir.name)\n self.eventbus.enabled = True\n\n @teardown\n def teardown(self):\n EventBus.shutdown()\n self.log_dir.cleanup()\n\n @mock.patch(\"tron.eventbus.reactor\", autospec=True)\n def test_start(self, reactor):\n self.eventbus.sync_load_log = mock.Mock()\n reactor.callLater = mock.Mock()\n self.eventbus.start()\n assert self.eventbus.sync_load_log.call_count == 1\n assert reactor.callLater.call_count == 1\n\n def test_shutdown(self):\n assert self.eventbus.enabled\n self.eventbus.sync_save_log = mock.Mock()\n self.eventbus.shutdown()\n assert not self.eventbus.enabled\n assert self.eventbus.sync_save_log.call_count == 1\n\n def test_publish(self):\n evt = {\"id\": \"foo\"}\n self.eventbus.publish(evt)\n assert self.eventbus.publish_queue.pop() == evt\n\n def test_subscribe(self):\n ps = (\"foo\", \"bar\", \"cb\")\n self.eventbus.subscribe(*ps)\n assert self.eventbus.subscribe_queue.pop() == ps\n\n def test_has_event(self):\n assert not self.eventbus.has_event(\"foo\")\n self.eventbus.event_log[\"foo\"] = \"bar\"\n assert self.eventbus.has_event(\"foo\")\n\n @mock.patch(\"tron.eventbus.time\", autospec=True)\n def test_sync_load_log(self, time):\n time.time = mock.Mock(return_value=1.0)\n self.eventbus.event_log = {\"foo\": \"bar\"}\n self.eventbus.sync_save_log(\"test\")\n self.eventbus.event_log = {}\n self.eventbus.sync_load_log()\n assert self.eventbus.event_log == {\"foo\": \"bar\"}\n\n @mock.patch(\"tron.eventbus.time\", autospec=True)\n def test_sync_save_log_time(self, time):\n time.time = mock.Mock(return_value=1.0)\n self.eventbus.sync_save_log(\"test\")\n current_link = os.readlink(self.eventbus.log_current)\n assert_equal(current_link, os.path.join(self.log_dir.name, \"1.pickle\"))\n\n time.time = mock.Mock(return_value=2.0)\n self.eventbus.sync_save_log(\"test\")\n new_link = os.readlink(self.eventbus.log_current)\n assert_equal(new_link, os.path.join(self.log_dir.name, \"2.pickle\"))\n # we clean up the previous link so as not to have a million pickles\n # on disk\n assert not os.path.exists(current_link)\n # so at this point, we should only have the new link\n assert os.path.exists(new_link)\n\n @mock.patch(\"tron.eventbus.time\", autospec=True)\n @mock.patch(\"tron.eventbus.reactor\", autospec=True)\n def test_sync_loop(self, reactor, time):\n time.time = mock.Mock(return_value=0)\n reactor.callLater = mock.Mock()\n self.eventbus.enabled = True\n self.eventbus.sync_shutdown = mock.Mock()\n self.eventbus.sync_loop()\n assert reactor.callLater.call_count == 1\n assert self.eventbus.sync_shutdown.call_count == 0\n\n @mock.patch(\"tron.eventbus.reactor\", autospec=True)\n def test_sync_loop_shutdown(self, reactor):\n reactor.callLater = mock.Mock()\n self.eventbus.enabled = False\n self.eventbus.sync_save_log = mock.Mock()\n self.eventbus.sync_loop()\n assert reactor.callLater.call_count == 0\n\n @mock.patch(\"tron.eventbus.time\", autospec=True)\n def test_sync_process_save_log(self, time):\n time.time = mock.Mock(return_value=10)\n self.eventbus.log_updates = 1\n self.eventbus.log_last_save = 0\n self.eventbus.log_save_interval = 20\n self.eventbus.sync_save_log = mock.Mock()\n self.eventbus.sync_process()\n assert self.eventbus.sync_save_log.call_count == 0\n\n time.time = mock.Mock(return_value=21)\n self.eventbus.sync_process()\n assert self.eventbus.sync_save_log.call_count == 1\n\n time.time = mock.Mock(return_value=0)\n self.eventbus.log_updates = 0\n self.eventbus.log_save_updates = 20\n self.eventbus.sync_save_log = mock.Mock()\n self.eventbus.sync_process()\n assert self.eventbus.sync_save_log.call_count == 0\n\n self.eventbus.log_updates = 21\n self.eventbus.sync_process()\n assert self.eventbus.sync_save_log.call_count == 1\n assert self.eventbus.log_updates == 0\n\n @mock.patch(\"tron.eventbus.time\", autospec=True)\n def test_sync_process_flush_queues(self, time):\n time.time = mock.Mock(return_value=10)\n self.eventbus.sync_subscribe = mock.Mock()\n self.eventbus.sync_publish = mock.Mock()\n\n for _ in range(5):\n self.eventbus.publish_queue.append(mock.Mock())\n self.eventbus.subscribe_queue.append(mock.Mock())\n\n self.eventbus.sync_process()\n\n assert_equal(self.eventbus.sync_subscribe.call_count, 5)\n assert_equal(self.eventbus.sync_publish.call_count, 5)\n\n @mock.patch(\"tron.eventbus.reactor\", autospec=True)\n def test_sync_publish(self, reactor):\n reactor.callLater = mock.Mock()\n evt = {\"id\": \"foo\", \"bar\": \"baz\"}\n self.eventbus.event_log = {}\n self.eventbus.log_save_updates = 0\n self.eventbus.sync_publish(evt)\n assert self.eventbus.log_updates == 1\n assert reactor.callLater.call_count == 1\n\n @mock.patch(\"tron.eventbus.reactor\", autospec=True)\n def test_sync_publish_replace(self, reactor):\n evt1 = {\"id\": \"foo\", \"bar\": \"baz\"}\n evt2 = {\"id\": \"foo\", \"bar\": \"quux\"}\n self.eventbus.event_log = {}\n self.eventbus.log_save_updates = 0\n self.eventbus.sync_publish(evt1)\n self.eventbus.sync_publish(evt2)\n assert self.eventbus.log_updates == 2\n assert reactor.callLater.call_count == 2\n\n @mock.patch(\"tron.eventbus.reactor\", autospec=True)\n def test_sync_publish_duplicate(self, reactor):\n evt = {\"id\": \"foo\", \"bar\": \"baz\"}\n self.eventbus.event_log = {\"foo\": {\"bar\": \"baz\"}}\n self.eventbus.log_save_updates = 0\n self.eventbus.sync_publish(evt)\n assert self.eventbus.log_updates == 0\n assert reactor.callLater.call_count == 0\n\n def test_sync_subscribe(self):\n self.eventbus.event_subscribers = defaultdict(list)\n self.eventbus.sync_subscribe((\"pre\", \"sub\", \"cb\"))\n assert self.eventbus.event_subscribers == {\"pre\": [(\"sub\", \"cb\")]}\n\n self.eventbus.sync_subscribe((\"pre\", \"sub2\", \"cb2\"))\n assert self.eventbus.event_subscribers == {\n \"pre\": [(\"sub\", \"cb\"), (\"sub2\", \"cb2\")],\n }\n\n def test_sync_unsubscribe(self):\n self.eventbus.event_subscribers = defaultdict(list)\n self.eventbus.sync_subscribe((\"pre\", \"sub\", \"cb\"))\n self.eventbus.sync_subscribe((\"pre\", \"sub2\", \"cb2\"))\n assert self.eventbus.event_subscribers == {\n \"pre\": [(\"sub\", \"cb\"), (\"sub2\", \"cb2\")],\n }\n\n self.eventbus.sync_unsubscribe((\"pre\", \"sub\"))\n assert self.eventbus.event_subscribers == {\"pre\": [(\"sub2\", \"cb2\")]}\n self.eventbus.sync_unsubscribe((\"pre\", \"sub2\"))\n assert self.eventbus.event_subscribers == {}\n\n @mock.patch(\"tron.eventbus.reactor\", autospec=True)\n def test_sync_notify(self, reactor):\n reactor.callLater = mock.Mock()\n self.eventbus.event_log = {\"p\": {}, \"pre\": {}, \"prefix\": {}}\n self.eventbus.event_subscribers = {\n \"pre\": [(\"sub\", \"m1\")],\n \"prefix\": [(\"sub\", \"m2\"), (\"sub2\", \"m3\")],\n }\n\n self.eventbus.sync_notify(\"p\")\n assert reactor.callLater.call_count == 0\n\n self.eventbus.sync_notify(\"pre\")\n assert reactor.callLater.call_count == 1\n\n self.eventbus.sync_notify(\"prefix\")\n assert reactor.callLater.call_count == 4\n" }, { "path": "tests/kubernetes_test.py", "content": "from typing import Any\nfrom unittest import mock\n\nimport pytest\nfrom task_processing.interfaces.event import Event\nfrom task_processing.plugins.kubernetes.task_config import KubernetesTaskConfig\n\nfrom tron.config.schema import ConfigFieldSelectorSource\nfrom tron.config.schema import ConfigKubernetes\nfrom tron.config.schema import ConfigProjectedSAVolume\nfrom tron.config.schema import ConfigSecretSource\nfrom tron.config.schema import ConfigSecretVolume\nfrom tron.config.schema import ConfigSecretVolumeItem\nfrom tron.config.schema import ConfigVolume\nfrom tron.kubernetes import DEFAULT_DISK_LIMIT\nfrom tron.kubernetes import KubernetesCluster\nfrom tron.kubernetes import KubernetesClusterRepository\nfrom tron.kubernetes import KubernetesTask\nfrom tron.utils import exitcode\n\n\n@pytest.fixture\ndef mock_kubernetes_task():\n with mock.patch(\n \"tron.kubernetes.logging.getLogger\",\n return_value=mock.Mock(handlers=[mock.Mock()]),\n autospec=None,\n ):\n yield KubernetesTask(\n action_run_id=\"mock_service.mock_job.1.mock_action\",\n task_config=KubernetesTaskConfig(\n name=\"mock--service-mock-job-mock--action\", uuid=\"123456\", image=\"some_image\", command=\"echo test\"\n ),\n )\n\n\n@pytest.fixture\ndef mock_kubernetes_cluster():\n with mock.patch(\"tron.kubernetes.PyDeferredQueue\", autospec=True,), mock.patch(\n \"tron.kubernetes.TaskProcessor\",\n autospec=True,\n ), mock.patch(\n \"tron.kubernetes.Subscription\",\n autospec=True,\n ) as mock_runner:\n mock_runner.return_value.configure_mock(\n stopping=False, TASK_CONFIG_INTERFACE=mock.Mock(spec=KubernetesTaskConfig)\n )\n yield KubernetesCluster(\"kube-cluster-a:1234\")\n\n\n@pytest.fixture\ndef mock_disabled_kubernetes_cluster():\n with mock.patch(\"tron.kubernetes.PyDeferredQueue\", autospec=True,), mock.patch(\n \"tron.kubernetes.TaskProcessor\",\n autospec=True,\n ), mock.patch(\n \"tron.kubernetes.Subscription\",\n autospec=True,\n ):\n yield KubernetesCluster(\"kube-cluster-a:1234\", enabled=False)\n\n\ndef mock_event_factory(\n task_id: str,\n platform_type: str,\n message: str = None,\n raw: dict[str, Any] = None,\n success: bool = False,\n terminal: bool = False,\n) -> Event:\n return Event(\n kind=\"task\",\n task_id=task_id,\n platform_type=platform_type,\n raw=raw or {},\n terminal=terminal,\n success=success,\n message=message,\n )\n\n\ndef test_get_event_logger_add_unique_handlers(mock_kubernetes_task):\n \"\"\"\n Ensures that only a single handler (for stderr) is added to the\n Kubernetes Taskevent logger, to prevent duplicate log output.\n \"\"\"\n # Call 2 times to make sure 2nd call doesn't add another handler\n logger = mock_kubernetes_task.get_event_logger()\n logger = mock_kubernetes_task.get_event_logger()\n\n assert len(logger.handlers) == 1\n\n\ndef test_handle_event_log_event_info_exception(mock_kubernetes_task):\n with mock.patch.object(\n mock_kubernetes_task, \"log_event_info\", autospec=True, side_effect=Exception\n ) as mock_log_event_info:\n mock_kubernetes_task.handle_event(\n mock_event_factory(task_id=mock_kubernetes_task.get_kubernetes_id(), platform_type=\"running\")\n )\n\n # TODO: should also assert that the task is in the expected state once that's hooked up\n assert mock_log_event_info.called\n\n\ndef test_handle_event_exit_early_on_misrouted_event(mock_kubernetes_task):\n with mock.patch.object(\n mock_kubernetes_task,\n \"log_event_info\",\n autospec=True,\n ) as mock_log_event_info:\n mock_kubernetes_task.handle_event(\n mock_event_factory(task_id=\"not-the-pods-youre-looking-for\", platform_type=\"finished\")\n )\n\n # TODO: should also assert that the task is in the expected state once that's hooked up\n # we log before actually doing anything with an event, so this not being called means\n # we exited early\n assert not mock_log_event_info.called\n\n\ndef test_handle_event_running(mock_kubernetes_task):\n mock_kubernetes_task.handle_event(\n mock_event_factory(task_id=mock_kubernetes_task.get_kubernetes_id(), platform_type=\"running\")\n )\n\n assert mock_kubernetes_task.state == mock_kubernetes_task.RUNNING\n\n\ndef test_handle_event_exit_on_finished(mock_kubernetes_task):\n mock_kubernetes_task.started()\n raw_event_data = {\n \"status\": {\n \"containerStatuses\": [\n {\n \"containerID\": \"docker://asdf\",\n \"image\": \"someimage\",\n \"imageID\": \"docker-pullable://someimage:sometag\",\n \"lastState\": {\"running\": None, \"terminated\": None, \"waiting\": None},\n \"name\": \"main\",\n \"ready\": False,\n \"restartCount\": 0,\n \"started\": False,\n \"state\": {\n \"running\": None,\n \"terminated\": {\n \"containerID\": \"docker://asdf\",\n \"exitCode\": 0,\n \"finishedAt\": \"2022-11-19 00:11:02+00:00\",\n \"message\": None,\n \"reason\": \"Completed\",\n \"signal\": None,\n \"startedAt\": None,\n },\n \"waiting\": None,\n },\n },\n ],\n }\n }\n mock_kubernetes_task.handle_event(\n mock_event_factory(\n task_id=mock_kubernetes_task.get_kubernetes_id(),\n raw=raw_event_data,\n platform_type=\"finished\",\n terminal=True,\n success=True,\n )\n )\n assert mock_kubernetes_task.state == mock_kubernetes_task.COMPLETE\n assert mock_kubernetes_task.is_complete\n\n\ndef test_handle_event_exit_on_failed(mock_kubernetes_task):\n mock_kubernetes_task.started()\n mock_kubernetes_task.handle_event(\n mock_event_factory(\n task_id=mock_kubernetes_task.get_kubernetes_id(), platform_type=\"failed\", terminal=True, success=False\n )\n )\n\n assert mock_kubernetes_task.is_failed\n assert mock_kubernetes_task.is_done\n\n\ndef test_handle_event_spot_interruption_exit(mock_kubernetes_task):\n mock_kubernetes_task.started()\n raw_event_data = {\n \"status\": {\n \"containerStatuses\": [\n {\n \"containerID\": None,\n \"image\": \"someimage\",\n \"imageID\": None,\n \"lastState\": {\n \"running\": None,\n \"terminated\": {\n \"containerID\": None,\n \"exitCode\": 137,\n \"finishedAt\": None,\n \"message\": \"The container could not be located when the pod was deleted. The container used to be Running\",\n \"reason\": \"ContainerStatusUnknown\",\n \"signal\": None,\n \"startedAt\": None,\n },\n \"waiting\": None,\n },\n \"name\": \"main\",\n \"ready\": False,\n \"restartCount\": 0,\n \"started\": False,\n \"state\": {\n \"running\": None,\n \"terminated\": None,\n \"waiting\": {\"message\": None, \"reason\": \"ContainerCreating\"},\n },\n },\n ],\n }\n }\n mock_kubernetes_task.handle_event(\n mock_event_factory(\n task_id=mock_kubernetes_task.get_kubernetes_id(),\n raw=raw_event_data,\n platform_type=\"killed\",\n terminal=True,\n success=False,\n )\n )\n assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_SPOT_INTERRUPTION\n assert mock_kubernetes_task.is_failed\n assert mock_kubernetes_task.is_done\n\n # Test again, but with no lastState\n raw_event_data[\"status\"][\"containerStatuses\"][0][\"state\"][\"terminated\"] = raw_event_data[\"status\"][\n \"containerStatuses\"\n ][0][\"lastState\"][\"terminated\"]\n raw_event_data[\"status\"][\"containerStatuses\"][0][\"lastState\"] = {}\n\n mock_kubernetes_task.handle_event(\n mock_event_factory(\n task_id=mock_kubernetes_task.get_kubernetes_id(),\n raw=raw_event_data,\n platform_type=\"killed\",\n terminal=True,\n success=False,\n )\n )\n assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_SPOT_INTERRUPTION\n assert mock_kubernetes_task.is_failed\n assert mock_kubernetes_task.is_done\n\n\ndef test_handle_event_node_scaledown_exit(mock_kubernetes_task):\n mock_kubernetes_task.started()\n raw_event_data = {\n \"status\": {\n \"containerStatuses\": [\n {\n \"containerID\": \"docker://asdf\",\n \"image\": \"someimage\",\n \"imageID\": \"docker-pullable://someimage:sometag\",\n \"lastState\": {\"running\": None, \"terminated\": None, \"waiting\": None},\n \"name\": \"main\",\n \"ready\": False,\n \"restartCount\": 0,\n \"started\": False,\n \"state\": {\n \"running\": None,\n \"terminated\": {\n \"containerID\": \"docker://asdf\",\n \"exitCode\": 143,\n \"finishedAt\": \"2022-11-19 00:11:02+00:00\",\n \"message\": None,\n \"reason\": \"Error\",\n \"signal\": None,\n \"startedAt\": None,\n },\n \"waiting\": None,\n },\n },\n ],\n }\n }\n mock_kubernetes_task.handle_event(\n mock_event_factory(\n task_id=mock_kubernetes_task.get_kubernetes_id(),\n raw=raw_event_data,\n platform_type=\"failed\",\n terminal=True,\n success=False,\n )\n )\n assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_NODE_SCALEDOWN\n assert mock_kubernetes_task.is_failed\n assert mock_kubernetes_task.is_done\n\n\ndef test_handle_event_exit_not_terminated(mock_kubernetes_task):\n mock_kubernetes_task.started()\n raw_event_data = {\n \"status\": {\n \"containerStatuses\": [\n {\n \"containerID\": \"docker://asdf\",\n \"image\": \"someimage\",\n \"imageID\": \"docker-pullable://someimage:sometag\",\n \"lastState\": {},\n \"name\": \"main\",\n \"ready\": False,\n \"restartCount\": 0,\n \"started\": False,\n \"state\": {\n \"running\": None,\n \"terminated\": None,\n \"waiting\": {\"reason\": \"ContainerCreating\"},\n },\n },\n ],\n }\n }\n mock_kubernetes_task.handle_event(\n mock_event_factory(\n task_id=mock_kubernetes_task.get_kubernetes_id(),\n raw=raw_event_data,\n platform_type=\"killed\",\n terminal=True,\n success=False,\n )\n )\n\n assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_NODE_SCALEDOWN\n assert mock_kubernetes_task.is_failed\n assert mock_kubernetes_task.is_done\n\n\ndef test_handle_event_abnormal_exit(mock_kubernetes_task):\n mock_kubernetes_task.started()\n raw_event_data = {\n \"status\": {\n \"containerStatuses\": [\n {\n \"containerID\": \"docker://asdf\",\n \"image\": \"someimage\",\n \"imageID\": \"docker-pullable://someimage:sometag\",\n \"lastState\": {\"running\": None, \"terminated\": None, \"waiting\": None},\n \"name\": \"main\",\n \"ready\": False,\n \"restartCount\": 0,\n \"started\": False,\n \"state\": {\n \"running\": None,\n \"terminated\": {\n \"containerID\": \"docker://asdf\",\n \"exitCode\": 0,\n \"finishedAt\": None,\n \"message\": None,\n \"reason\": None,\n \"signal\": None,\n \"startedAt\": None,\n },\n \"waiting\": None,\n },\n },\n ],\n }\n }\n mock_kubernetes_task.handle_event(\n mock_event_factory(\n task_id=mock_kubernetes_task.get_kubernetes_id(),\n raw=raw_event_data,\n platform_type=\"finished\",\n terminal=True,\n success=False,\n )\n )\n assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_ABNORMAL\n assert mock_kubernetes_task.is_failed\n assert mock_kubernetes_task.is_done\n\n\ndef test_handle_event_missing_state(mock_kubernetes_task):\n mock_kubernetes_task.started()\n raw_event_data = {\n \"status\": {\n \"containerStatuses\": [\n {\n \"containerID\": \"docker://asdf\",\n \"image\": \"someimage\",\n \"imageID\": \"docker-pullable://someimage:sometag\",\n \"lastState\": {},\n \"name\": \"main\",\n \"ready\": False,\n \"restartCount\": 0,\n \"started\": False,\n \"state\": None,\n },\n ],\n }\n }\n mock_kubernetes_task.handle_event(\n mock_event_factory(\n task_id=mock_kubernetes_task.get_kubernetes_id(),\n raw=raw_event_data,\n platform_type=\"killed\",\n terminal=True,\n success=False,\n )\n )\n assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_ABNORMAL\n assert mock_kubernetes_task.is_failed\n assert mock_kubernetes_task.is_done\n\n\ndef test_handle_event_code_from_state(mock_kubernetes_task):\n mock_kubernetes_task.started()\n raw_event_data = {\n \"status\": {\n \"containerStatuses\": [\n {\n \"containerID\": \"docker://asdf\",\n \"image\": \"someimage\",\n \"imageID\": \"docker-pullable://someimage:sometag\",\n \"lastState\": {},\n \"name\": \"main\",\n \"ready\": False,\n \"restartCount\": 0,\n \"started\": False,\n \"state\": {\n \"running\": None,\n \"terminated\": {\n \"containerID\": \"docker://asdf\",\n \"exitCode\": 1337,\n \"finishedAt\": None,\n \"message\": None,\n \"reason\": None,\n \"signal\": None,\n \"startedAt\": None,\n },\n \"waiting\": None,\n },\n },\n ],\n }\n }\n mock_kubernetes_task.handle_event(\n mock_event_factory(\n task_id=mock_kubernetes_task.get_kubernetes_id(),\n raw=raw_event_data,\n platform_type=\"failed\",\n terminal=True,\n success=False,\n )\n )\n assert mock_kubernetes_task.exit_status == 1337\n assert mock_kubernetes_task.is_failed\n assert mock_kubernetes_task.is_done\n\n\ndef test_handle_event_lost(mock_kubernetes_task):\n mock_kubernetes_task.started()\n mock_kubernetes_task.handle_event(\n mock_event_factory(\n task_id=mock_kubernetes_task.get_kubernetes_id(),\n platform_type=\"lost\",\n )\n )\n\n assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_TASK_LOST\n\n\ndef test_create_task_disabled():\n cluster = KubernetesCluster(\"kube-cluster-a:1234\", enabled=False)\n mock_serializer = mock.MagicMock()\n\n task = cluster.create_task(\n action_run_id=\"action_a\",\n serializer=mock_serializer,\n command=\"ls\",\n cpus=1,\n mem=1024,\n disk=None,\n docker_image=\"docker-paasta.yelpcorp.com:443/bionic_yelp\",\n env={},\n secret_env={},\n secret_volumes=[],\n projected_sa_volumes=[],\n field_selector_env={},\n volumes=[],\n cap_add=[],\n cap_drop=[],\n node_selectors={\"yelp.com/pool\": \"default\"},\n node_affinities=[],\n topology_spread_constraints=[],\n pod_labels={},\n pod_annotations={},\n service_account_name=None,\n ports=[],\n )\n\n assert task is None\n\n\ndef test_create_task(mock_kubernetes_cluster):\n mock_serializer = mock.MagicMock()\n\n task = mock_kubernetes_cluster.create_task(\n action_run_id=\"action_a\",\n serializer=mock_serializer,\n command=\"ls\",\n cpus=1,\n mem=1024,\n disk=None,\n docker_image=\"docker-paasta.yelpcorp.com:443/bionic_yelp\",\n env={},\n secret_env={},\n secret_volumes=[],\n projected_sa_volumes=[],\n field_selector_env={},\n volumes=[],\n cap_add=[],\n cap_drop=[],\n node_selectors={\"yelp.com/pool\": \"default\"},\n node_affinities=[],\n topology_spread_constraints=[],\n pod_labels={},\n pod_annotations={},\n service_account_name=None,\n ports=[],\n )\n\n assert task is not None\n\n\ndef test_create_task_with_task_id(mock_kubernetes_cluster):\n mock_serializer = mock.MagicMock()\n\n task = mock_kubernetes_cluster.create_task(\n action_run_id=\"action_a\",\n serializer=mock_serializer,\n task_id=\"yay.1234\",\n command=\"ls\",\n cpus=1,\n mem=1024,\n disk=None,\n docker_image=\"docker-paasta.yelpcorp.com:443/bionic_yelp\",\n env={},\n secret_env={},\n secret_volumes=[],\n projected_sa_volumes=[],\n field_selector_env={},\n volumes=[],\n cap_add=[],\n cap_drop=[],\n node_selectors={\"yelp.com/pool\": \"default\"},\n node_affinities=[],\n topology_spread_constraints=[],\n pod_labels={},\n pod_annotations={},\n service_account_name=None,\n ports=[],\n )\n\n mock_kubernetes_cluster.runner.TASK_CONFIG_INTERFACE().set_pod_name.assert_called_once_with(\"yay.1234\")\n assert task is not None\n\n\ndef test_create_task_with_invalid_task_id(mock_kubernetes_cluster):\n mock_serializer = mock.MagicMock()\n\n with mock.patch.object(mock_kubernetes_cluster, \"runner\") as mock_runner:\n mock_runner.TASK_CONFIG_INTERFACE.return_value.set_pod_name = mock.MagicMock(side_effect=ValueError)\n task = mock_kubernetes_cluster.create_task(\n action_run_id=\"action_a\",\n serializer=mock_serializer,\n task_id=\"boo\",\n command=\"ls\",\n cpus=1,\n mem=1024,\n disk=None,\n docker_image=\"docker-paasta.yelpcorp.com:443/bionic_yelp\",\n env={},\n secret_env={},\n secret_volumes=[],\n projected_sa_volumes=[],\n field_selector_env={},\n volumes=[],\n cap_add=[],\n cap_drop=[],\n node_selectors={\"yelp.com/pool\": \"default\"},\n node_affinities=[],\n topology_spread_constraints=[],\n pod_labels={},\n pod_annotations={},\n service_account_name=None,\n ports=[],\n )\n\n assert task is None\n\n\ndef test_create_task_with_config(mock_kubernetes_cluster):\n # Validate we pass all expected args to taskproc\n default_volumes = [ConfigVolume(container_path=\"/nail/tmp\", host_path=\"/nail/tmp\", mode=\"RO\")]\n\n mock_kubernetes_cluster.default_volumes = default_volumes\n mock_serializer = mock.MagicMock()\n\n config_volumes = [ConfigVolume(container_path=\"/tmp\", host_path=\"/host\", mode=\"RO\")]\n config_secret_volumes = [\n ConfigSecretVolume(\n secret_volume_name=\"secretvolumename\",\n secret_name=\"secret\",\n container_path=\"/b\",\n default_mode=\"0644\",\n items=[ConfigSecretVolumeItem(key=\"key\", path=\"path\", mode=\"0755\")],\n ),\n ]\n config_secrets = {\"TEST_SECRET\": ConfigSecretSource(secret_name=\"tron-secret-test-secret--A\", key=\"secret_A\")}\n config_field_selector = {\"POD_IP\": ConfigFieldSelectorSource(field_path=\"status.podIP\")}\n config_sa_volumes = [ConfigProjectedSAVolume(audience=\"for.bar.com\", container_path=\"/var/run/secrets/whatever\")]\n\n expected_args = {\n \"name\": mock.ANY,\n \"command\": \"ls\",\n \"image\": \"docker-paasta.yelpcorp.com:443/bionic_yelp\",\n \"cpus\": 1,\n \"memory\": 1024,\n \"disk\": DEFAULT_DISK_LIMIT,\n \"environment\": {\"TEST_ENV\": \"foo\"},\n \"secret_environment\": {k: v._asdict() for k, v in config_secrets.items()},\n \"secret_volumes\": [v._asdict() for v in config_secret_volumes],\n \"projected_sa_volumes\": [v._asdict() for v in config_sa_volumes],\n \"field_selector_environment\": {k: v._asdict() for k, v in config_field_selector.items()},\n \"volumes\": [v._asdict() for v in default_volumes + config_volumes],\n \"cap_add\": [\"KILL\"],\n \"cap_drop\": [\"KILL\", \"CHOWN\"],\n \"node_selectors\": {\"yelp.com/pool\": \"default\"},\n \"node_affinities\": [],\n \"topology_spread_constraints\": [],\n \"labels\": {},\n \"annotations\": {},\n \"service_account_name\": None,\n \"ports\": [],\n }\n\n task = mock_kubernetes_cluster.create_task(\n action_run_id=\"action_a\",\n serializer=mock_serializer,\n task_id=\"yay.1234\",\n command=expected_args[\"command\"],\n cpus=expected_args[\"cpus\"],\n mem=expected_args[\"memory\"],\n disk=None,\n docker_image=expected_args[\"image\"],\n env=expected_args[\"environment\"],\n secret_env=config_secrets,\n secret_volumes=config_secret_volumes,\n projected_sa_volumes=config_sa_volumes,\n field_selector_env=config_field_selector,\n volumes=config_volumes,\n cap_add=[\"KILL\"],\n cap_drop=[\"KILL\", \"CHOWN\"],\n node_selectors={\"yelp.com/pool\": \"default\"},\n node_affinities=[],\n topology_spread_constraints=[],\n pod_labels={},\n pod_annotations={},\n service_account_name=None,\n ports=expected_args[\"ports\"],\n )\n\n assert task is not None\n mock_kubernetes_cluster.runner.TASK_CONFIG_INTERFACE.assert_called_once_with(**expected_args)\n\n\ndef test_process_event_task(mock_kubernetes_cluster):\n event = mock_event_factory(task_id=\"abc.123\", platform_type=\"mock_type\")\n mock_kubernetes_task = mock.MagicMock(spec_set=KubernetesTask)\n mock_kubernetes_task.get_kubernetes_id.return_value = \"abc.123\"\n mock_kubernetes_cluster.tasks[\"abc.123\"] = mock_kubernetes_task\n\n mock_kubernetes_cluster.process_event(event)\n\n mock_kubernetes_task.handle_event.assert_called_once_with(event)\n\n\ndef test_process_event_task_invalid_id(mock_kubernetes_cluster):\n event = mock_event_factory(task_id=\"hwat.dis\", platform_type=\"mock_type\")\n mock_kubernetes_task = mock.MagicMock(spec_set=KubernetesTask)\n mock_kubernetes_task.get_kubernetes_id.return_value = \"abc.123\"\n mock_kubernetes_cluster.tasks[\"abc.123\"] = mock_kubernetes_task\n\n mock_kubernetes_cluster.process_event(event)\n\n assert mock_kubernetes_task.handle_event.call_count == 0\n\n\ndef test_stop_default(mock_kubernetes_cluster):\n # When stopping, tasks should not exit. They will be recovered\n mock_task = mock.MagicMock()\n mock_kubernetes_cluster.tasks = {\"task_id\": mock_task}\n mock_kubernetes_cluster.stop()\n assert mock_kubernetes_cluster.deferred is None\n assert mock_task.exited.call_count == 0\n assert len(mock_kubernetes_cluster.tasks) == 1\n\n\ndef test_stop_disabled():\n # Shouldn't raise an error\n mock_kubernetes_cluster = KubernetesCluster(\"kube-cluster-a:1234\", enabled=False)\n mock_kubernetes_cluster.stop()\n\n\ndef test_set_enabled_enable_already_on(mock_kubernetes_cluster):\n mock_kubernetes_cluster.set_enabled(is_enabled=True)\n\n assert mock_kubernetes_cluster.enabled is True\n # only called once as part of creating the cluster object\n mock_kubernetes_cluster.processor.executor_from_config.assert_called_once()\n assert mock_kubernetes_cluster.runner is not None\n assert mock_kubernetes_cluster.deferred is not None\n mock_kubernetes_cluster.deferred.addCallback.assert_has_calls(\n [\n mock.call(mock_kubernetes_cluster.process_event),\n mock.call(mock_kubernetes_cluster.handle_next_event),\n ]\n )\n\n\ndef test_set_enabled_enable(mock_disabled_kubernetes_cluster):\n mock_disabled_kubernetes_cluster.set_enabled(is_enabled=True)\n\n assert mock_disabled_kubernetes_cluster.enabled is True\n # only called once as part of enabling\n mock_disabled_kubernetes_cluster.processor.executor_from_config.assert_called_once()\n assert mock_disabled_kubernetes_cluster.runner is not None\n assert mock_disabled_kubernetes_cluster.deferred is not None\n mock_disabled_kubernetes_cluster.deferred.addCallback.assert_has_calls(\n [\n mock.call(mock_disabled_kubernetes_cluster.process_event),\n mock.call(mock_disabled_kubernetes_cluster.handle_next_event),\n ]\n )\n\n\ndef test_set_enabled_disable(mock_kubernetes_cluster):\n mock_task = mock.Mock(spec=KubernetesTask)\n mock_kubernetes_cluster.tasks == {\"a.b\": mock_task}\n\n mock_kubernetes_cluster.set_enabled(is_enabled=False)\n\n assert mock_kubernetes_cluster.enabled is False\n mock_kubernetes_cluster.runner.stop.assert_called_once()\n assert mock_kubernetes_cluster.deferred is None\n assert mock_kubernetes_cluster.tasks == {}\n\n\ndef test_configure_default_volumes():\n # default_volume validation is done at config time, we just need to validate we are setting it\n with mock.patch(\"tron.kubernetes.PyDeferredQueue\", autospec=True,), mock.patch(\n \"tron.kubernetes.TaskProcessor\",\n autospec=True,\n ), mock.patch(\n \"tron.kubernetes.Subscription\",\n autospec=True,\n ):\n mock_kubernetes_cluster = KubernetesCluster(\"kube-cluster-a:1234\", default_volumes=[])\n assert mock_kubernetes_cluster.default_volumes == []\n expected_volumes = [\n ConfigVolume(\n container_path=\"/tmp\",\n host_path=\"/host/tmp\",\n mode=\"RO\",\n ),\n ]\n mock_kubernetes_cluster.configure_tasks(default_volumes=expected_volumes)\n assert mock_kubernetes_cluster.default_volumes == expected_volumes\n\n\ndef test_submit_disabled(mock_disabled_kubernetes_cluster, mock_kubernetes_task):\n with mock.patch.object(mock_kubernetes_task, \"exited\", autospec=True) as mock_exited:\n mock_disabled_kubernetes_cluster.submit(mock_kubernetes_task)\n\n assert mock_kubernetes_task.get_kubernetes_id() not in mock_disabled_kubernetes_cluster.tasks\n mock_exited.assert_called_once_with(1)\n\n\ndef test_submit(mock_kubernetes_cluster, mock_kubernetes_task):\n mock_kubernetes_cluster.submit(mock_kubernetes_task)\n\n assert mock_kubernetes_task.get_kubernetes_id() in mock_kubernetes_cluster.tasks\n assert mock_kubernetes_cluster.tasks[mock_kubernetes_task.get_kubernetes_id()] == mock_kubernetes_task\n mock_kubernetes_cluster.runner.run.assert_called_once_with(mock_kubernetes_task.get_config())\n\n\ndef test_recover(mock_kubernetes_cluster, mock_kubernetes_task):\n with mock.patch.object(mock_kubernetes_task, \"started\", autospec=True) as mock_started:\n mock_kubernetes_cluster.recover(mock_kubernetes_task)\n\n assert mock_kubernetes_task.get_kubernetes_id() in mock_kubernetes_cluster.tasks\n mock_kubernetes_cluster.runner.reconcile.assert_called_once_with(mock_kubernetes_task.get_config())\n assert mock_started.call_count == 1\n\n\ndef test_kuberntes_cluster_repository():\n # Check we are passing k8s_options from mcp/KubernetesClusterRepository.configure to KubernetesCluster calls\n\n mock_k8s_options = {\n \"enabled\": True,\n \"kubeconfig_path\": \"/tmp/kubeconfig.conf\",\n \"watcher_kubeconfig_paths\": [\"/tmp/kubeconfig_old.conf\"],\n \"non_retryable_exit_codes\": [13],\n \"default_volumes\": [\n ConfigVolume(\n container_path=\"/tmp\",\n host_path=\"/host/tmp\",\n mode=\"RO\",\n )\n ],\n }\n mock_k8s_options_obj = ConfigKubernetes(**mock_k8s_options)\n\n with mock.patch(\"tron.kubernetes.KubernetesCluster\", autospec=True) as mock_cluster:\n KubernetesClusterRepository.configure(mock_k8s_options_obj)\n KubernetesClusterRepository.get_cluster()\n\n mock_cluster.assert_called_once_with(**mock_k8s_options)\n" }, { "path": "tests/mcp_reconfigure_test.py", "content": "\"\"\"Tests for reconfiguring mcp.\"\"\"\nimport os\nimport tempfile\nimport time\n\nimport pytest\n\nfrom testifycompat import assert_equal\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import suite\nfrom testifycompat import teardown\nfrom testifycompat import TestCase\nfrom tests.assertions import assert_length\nfrom tron import mcp\nfrom tron.config import config_parse\nfrom tron.config import schema\nfrom tron.serialize import filehandler\n\n\nclass TestMCPReconfigure(TestCase):\n\n os.environ[\"SSH_AUTH_SOCK\"] = \"test-socket\"\n pre_config = dict(\n ssh_options=dict(\n agent=True,\n identities=[\"tests/test_id_rsa\"],\n ),\n nodes=[\n dict(name=\"node0\", hostname=\"batch0\"),\n dict(name=\"node1\", hostname=\"batch1\"),\n ],\n node_pools=[dict(name=\"nodePool\", nodes=[\"node0\", \"node1\"])],\n command_context={\n \"thischanges\": \"froma\",\n },\n jobs=[\n dict(\n name=\"test_unchanged\",\n node=\"node0\",\n schedule=\"daily\",\n actions=[\n dict(\n name=\"action_unchanged\",\n command=\"command_unchanged\",\n ),\n ],\n ),\n dict(\n name=\"test_remove\",\n node=\"node1\",\n schedule={\"type\": \"cron\", \"value\": \"* * * * *\"},\n actions=[\n dict(\n name=\"action_remove\",\n command=\"command_remove\",\n ),\n ],\n cleanup_action=dict(name=\"cleanup\", command=\"doit\"),\n ),\n dict(\n name=\"test_change\",\n node=\"nodePool\",\n schedule={\"type\": \"cron\", \"value\": \"* * * * *\"},\n actions=[\n dict(\n name=\"action_change\",\n command=\"command_change\",\n ),\n dict(\n name=\"action_remove2\",\n command=\"command_remove2\",\n requires=[\"action_change\"],\n ),\n ],\n ),\n dict(\n name=\"test_daily_change\",\n node=\"node0\",\n schedule=\"daily\",\n actions=[\n dict(\n name=\"action_daily_change\",\n command=\"command\",\n ),\n ],\n ),\n dict(\n name=\"test_action_added\",\n node=\"node0\",\n schedule={\"type\": \"cron\", \"value\": \"* * * * *\"},\n actions=[\n dict(name=\"action_first\", command=\"command_do_it\"),\n ],\n ),\n ],\n )\n\n post_config = dict(\n ssh_options=dict(\n agent=True,\n identities=[\"tests/test_id_rsa\"],\n ),\n nodes=[\n dict(name=\"node0\", hostname=\"batch0\"),\n dict(name=\"node1\", hostname=\"batch1\"),\n ],\n node_pools=[dict(name=\"nodePool\", nodes=[\"node0\", \"node1\"])],\n command_context={\n \"a_variable\": \"is_constant\",\n \"thischanges\": \"tob\",\n },\n jobs=[\n dict(\n name=\"test_unchanged\",\n node=\"node0\",\n schedule=\"daily\",\n actions=[\n dict(\n name=\"action_unchanged\",\n command=\"command_unchanged\",\n ),\n ],\n ),\n dict(\n name=\"test_change\",\n node=\"nodePool\",\n schedule=\"daily\",\n actions=[\n dict(\n name=\"action_change\",\n command=\"command_changed\",\n ),\n ],\n ),\n dict(\n name=\"test_daily_change\",\n node=\"node0\",\n schedule=\"daily\",\n actions=[\n dict(\n name=\"action_daily_change\",\n command=\"command_changed\",\n ),\n ],\n ),\n dict(\n name=\"test_new\",\n node=\"nodePool\",\n schedule={\"type\": \"cron\", \"value\": \"* * * * *\"},\n actions=[\n dict(\n name=\"action_new\",\n command=\"command_new\",\n ),\n ],\n ),\n dict(\n name=\"test_action_added\",\n node=\"node0\",\n schedule={\"type\": \"cron\", \"value\": \"* * * * *\"},\n actions=[\n dict(name=\"action_first\", command=\"command_do_it\"),\n dict(name=\"action_second\", command=\"command_ok\"),\n ],\n ),\n ],\n )\n\n def _get_config(self, idx, output_dir):\n config = dict(self.post_config if idx else self.pre_config)\n config[\"output_stream_dir\"] = output_dir\n return config\n\n def _get_runs_to_schedule(self, sched):\n last_run = sched.job.runs.get_newest(include_manual=False)\n last_run_time = last_run.run_time if last_run else None\n return sched.get_runs_to_schedule(last_run_time)\n\n @setup\n def setup_mcp(self):\n self.test_dir = tempfile.mkdtemp()\n self.mcp = mcp.MasterControlProgram(self.test_dir, \"config\", time.time())\n config = {schema.MASTER_NAMESPACE: self._get_config(0, self.test_dir)}\n container = config_parse.ConfigContainer.create(config)\n self.mcp.apply_config(container)\n\n @teardown\n def teardown_mcp(self):\n filehandler.OutputPath(self.test_dir).delete()\n filehandler.FileHandleManager.reset()\n\n def reconfigure(self):\n config = {schema.MASTER_NAMESPACE: self._get_config(1, self.test_dir)}\n container = config_parse.ConfigContainer.create(config)\n self.mcp.apply_config(container, reconfigure=True)\n\n @suite(\"integration\")\n def test_job_list(self):\n count = len(self.pre_config[\"jobs\"])\n assert_equal(len(self.mcp.jobs.get_names()), count)\n self.reconfigure()\n assert_equal(len(self.mcp.jobs.get_names()), count)\n\n @pytest.mark.skip(\n reason=\"This test doesn't currently as run1 is not scheduled.\",\n )\n @suite(\"integration\")\n def test_job_unchanged(self):\n assert \"MASTER.test_unchanged\" in self.mcp.jobs\n job_sched = self.mcp.jobs.get_by_name(\"MASTER.test_unchanged\")\n orig_job = job_sched.job\n run0 = next(self._get_runs_to_schedule(job_sched))\n run0.start()\n run1 = next(self._get_runs_to_schedule(job_sched))\n\n assert_equal(job_sched.job.name, \"MASTER.test_unchanged\")\n action_map = job_sched.job.action_graph.action_map\n assert_equal(len(action_map), 1)\n assert_equal(action_map[\"action_unchanged\"].name, \"action_unchanged\")\n assert_equal(str(job_sched.job.scheduler), \"daily 00:00:00 \")\n\n self.reconfigure()\n assert job_sched is self.mcp.jobs.get_by_name(\"MASTER.test_unchanged\")\n assert job_sched.job is orig_job\n\n assert_equal(len(job_sched.job.runs.runs), 2)\n assert_equal(job_sched.job.runs.runs[1], run0)\n assert_equal(job_sched.job.runs.runs[0], run1)\n assert run1.is_scheduled\n assert_equal(job_sched.job.context[\"a_variable\"], \"is_constant\")\n assert_equal(job_sched.job.context[\"thischanges\"], \"tob\")\n\n @suite(\"integration\")\n def test_job_unchanged_disabled(self):\n job_sched = self.mcp.jobs.get_by_name(\"MASTER.test_unchanged\")\n orig_job = job_sched.job\n next(self._get_runs_to_schedule(job_sched))\n job_sched.disable()\n\n self.reconfigure()\n assert job_sched is self.mcp.jobs.get_by_name(\"MASTER.test_unchanged\")\n assert job_sched.job is orig_job\n assert not job_sched.job.enabled\n\n @suite(\"integration\")\n def test_job_removed(self):\n assert \"MASTER.test_remove\" in self.mcp.jobs\n job_sched = self.mcp.jobs.get_by_name(\"MASTER.test_remove\")\n run0 = next(self._get_runs_to_schedule(job_sched))\n run0.start()\n run1 = next(self._get_runs_to_schedule(job_sched))\n\n assert_equal(job_sched.job.name, \"MASTER.test_remove\")\n action_map = job_sched.job.action_graph.action_map\n assert_equal(len(action_map), 2)\n assert_equal(action_map[\"action_remove\"].name, \"action_remove\")\n\n self.reconfigure()\n assert \"test_remove\" not in self.mcp.jobs\n assert not job_sched.job.enabled\n assert not run1.is_scheduled\n\n @suite(\"integration\")\n def test_job_changed(self):\n assert \"MASTER.test_change\" in self.mcp.jobs\n job_sched = self.mcp.jobs.get_by_name(\"MASTER.test_change\")\n run0 = next(self._get_runs_to_schedule(job_sched))\n run0.start()\n next(self._get_runs_to_schedule(job_sched))\n assert_equal(len(job_sched.job.runs.runs), 2)\n\n assert_equal(job_sched.job.name, \"MASTER.test_change\")\n action_map = job_sched.job.action_graph.action_map\n assert_equal(len(action_map), 2)\n\n self.reconfigure()\n new_job_sched = self.mcp.jobs.get_by_name(\"MASTER.test_change\")\n assert new_job_sched is job_sched\n assert new_job_sched.job is job_sched.job\n\n assert_equal(new_job_sched.job.name, \"MASTER.test_change\")\n action_map = job_sched.job.action_graph.action_map\n assert_equal(len(action_map), 1)\n\n assert_equal(len(new_job_sched.job.runs.runs), 2)\n assert new_job_sched.job.runs.runs[1].is_starting\n assert new_job_sched.job.runs.runs[0].is_scheduled\n assert_equal(job_sched.job.context[\"a_variable\"], \"is_constant\")\n assert new_job_sched.job.context.base.job is new_job_sched.job\n\n @suite(\"integration\")\n def test_job_changed_disabled(self):\n job_sched = self.mcp.jobs.get_by_name(\"MASTER.test_change\")\n job_sched.disable()\n assert not job_sched.job.enabled\n\n self.reconfigure()\n new_job_sched = self.mcp.jobs.get_by_name(\"MASTER.test_change\")\n assert not new_job_sched.job.enabled\n\n @suite(\"integration\")\n def test_job_new(self):\n assert \"test_new\" not in self.mcp.jobs\n self.reconfigure()\n\n assert \"MASTER.test_new\" in self.mcp.jobs\n job_sched = self.mcp.jobs.get_by_name(\"MASTER.test_new\")\n\n assert_equal(job_sched.job.name, \"MASTER.test_new\")\n action_map = job_sched.job.action_graph.action_map\n assert_equal(len(action_map), 1)\n assert_equal(action_map[\"action_new\"].name, \"action_new\")\n assert_equal(action_map[\"action_new\"].command, \"command_new\")\n assert_equal(len(job_sched.job.runs.runs), 1)\n assert job_sched.job.runs.runs[0].is_scheduled\n\n @suite(\"integration\")\n def test_daily_reschedule(self):\n job_sched = self.mcp.jobs.get_by_name(\"MASTER.test_daily_change\")\n\n next(self._get_runs_to_schedule(job_sched))\n\n assert_equal(len(job_sched.job.runs.runs), 1)\n run = job_sched.job.runs.runs[0]\n assert run.is_scheduled\n\n action_runs = run.action_runs\n self.reconfigure()\n assert action_runs.is_cancelled\n\n assert_equal(len(job_sched.job.runs.runs), 1)\n new_run = job_sched.job.runs.runs[0]\n assert new_run is not run\n assert new_run.is_scheduled\n assert_equal(run.run_time, new_run.run_time)\n\n @suite(\"integration\")\n def test_action_added(self):\n self.reconfigure()\n job_sched = self.mcp.jobs.get_by_name(\"MASTER.test_action_added\")\n assert_length(job_sched.job.action_graph.action_map, 2)\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/mcp_test.py", "content": "import shutil\nimport tempfile\nimport time\nfrom unittest import mock\n\nimport pytest\n\nfrom testifycompat import assert_equal\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import teardown\nfrom testifycompat import TestCase\nfrom tests.testingutils import autospec_method\nfrom tron import mcp\nfrom tron.config import config_parse\nfrom tron.config import manager\nfrom tron.core.job_collection import JobCollection\nfrom tron.serialize.runstate import statemanager\n\n\nclass TestMasterControlProgram:\n\n TEST_CONFIG = \"tests/data/test_config.yaml\"\n\n @pytest.fixture(autouse=True)\n def setup_mcp(self):\n self.working_dir = tempfile.mkdtemp()\n self.config_path = tempfile.mkdtemp()\n self.boot_time = time.time()\n self.mcp = mcp.MasterControlProgram(self.working_dir, self.config_path, self.boot_time)\n self.mcp.state_watcher = mock.create_autospec(\n statemanager.StateChangeWatcher,\n )\n yield\n shutil.rmtree(self.config_path)\n shutil.rmtree(self.working_dir)\n\n def test_reconfigure_default(self):\n autospec_method(self.mcp._load_config)\n self.mcp.state_watcher = mock.MagicMock()\n self.mcp.reconfigure()\n self.mcp._load_config.assert_called_with(reconfigure=True, namespace_to_reconfigure=None)\n\n def test_reconfigure_namespace(self):\n autospec_method(self.mcp._load_config)\n self.mcp.state_watcher = mock.MagicMock()\n self.mcp.reconfigure(namespace=\"foo\")\n self.mcp._load_config.assert_called_with(reconfigure=True, namespace_to_reconfigure=\"foo\")\n\n @pytest.mark.parametrize(\n \"reconfigure,namespace\",\n [\n (False, None),\n (True, None),\n (True, \"foo\"),\n ],\n )\n def test_load_config(self, reconfigure, namespace):\n autospec_method(self.mcp.apply_config)\n self.mcp.config = mock.create_autospec(manager.ConfigManager)\n self.mcp._load_config(reconfigure, namespace)\n self.mcp.state_watcher.disabled.assert_called_with()\n self.mcp.apply_config.assert_called_with(\n self.mcp.config.load.return_value,\n reconfigure=reconfigure,\n namespace_to_reconfigure=namespace,\n )\n\n @pytest.mark.parametrize(\n \"reconfigure,namespace\",\n [\n (False, None),\n (True, None),\n (True, \"foo\"),\n (True, \"MASTER\"),\n ],\n )\n @mock.patch(\"tron.mcp.KubernetesClusterRepository\", autospec=True)\n @mock.patch(\"tron.mcp.MesosClusterRepository\", autospec=True)\n @mock.patch(\"tron.mcp.node.NodePoolRepository\", autospec=True)\n def test_apply_config(self, mock_repo, mock_cluster_repo, mock_k8s_cluster_repo, reconfigure, namespace):\n config_container = mock.create_autospec(config_parse.ConfigContainer)\n master_config = config_container.get_master.return_value\n autospec_method(self.mcp.jobs.update_from_config)\n autospec_method(self.mcp.build_job_scheduler_factory)\n self.mcp.apply_config(config_container, reconfigure, namespace)\n self.mcp.state_watcher.update_from_config.assert_called_with(\n master_config.state_persistence,\n )\n assert_equal(self.mcp.context.base, master_config.command_context)\n\n mock_repo.update_from_config.assert_called_with(\n master_config.nodes,\n master_config.node_pools,\n master_config.ssh_options,\n )\n mock_cluster_repo.configure.assert_called_with(\n master_config.mesos_options,\n )\n mock_k8s_cluster_repo.configure.assert_called_with(\n master_config.k8s_options,\n )\n self.mcp.build_job_scheduler_factory(master_config, mock.Mock())\n\n expected_namespace_to_update = None if namespace == \"MASTER\" else namespace\n self.mcp.jobs.update_from_config.assert_called_once_with(\n config_container.get_jobs(),\n self.mcp.build_job_scheduler_factory.return_value,\n reconfigure,\n expected_namespace_to_update,\n )\n self.mcp.state_watcher.watch_all.assert_called_once_with(\n self.mcp.jobs.update_from_config.return_value,\n mock.ANY,\n )\n\n def test_update_state_watcher_config_changed(self):\n self.mcp.state_watcher.update_from_config.return_value = True\n self.mcp.jobs = mock.create_autospec(JobCollection)\n self.mcp.jobs.__iter__.return_values = {\n \"a\": mock.Mock(),\n \"b\": mock.Mock(),\n }\n state_config = mock.Mock()\n self.mcp.update_state_watcher_config(state_config)\n self.mcp.state_watcher.update_from_config.assert_called_with(\n state_config,\n )\n assert_equal(\n self.mcp.state_watcher.save_job.mock_calls,\n [mock.call(j.job) for j in self.mcp.jobs],\n )\n\n def test_update_state_watcher_config_no_change(self):\n self.mcp.state_watcher.update_from_config.return_value = False\n self.mcp.jobs = {\"a\": mock.Mock(), \"b\": mock.Mock()}\n state_config = mock.Mock()\n self.mcp.update_state_watcher_config(state_config)\n assert not self.mcp.state_watcher.save_job.mock_calls\n\n\nclass TestMasterControlProgramRestoreState(TestCase):\n @setup\n def setup_mcp(self):\n self.working_dir = tempfile.mkdtemp()\n self.config_path = tempfile.mkdtemp()\n self.boot_time = time.time()\n self.mcp = mcp.MasterControlProgram(self.working_dir, self.config_path, self.boot_time)\n self.mcp.jobs = mock.create_autospec(JobCollection)\n self.mcp.state_watcher = mock.create_autospec(\n statemanager.StateChangeWatcher,\n )\n\n @teardown\n def teardown_mcp(self):\n shutil.rmtree(self.working_dir)\n shutil.rmtree(self.config_path)\n\n def test_restore_state(self):\n job_state_data = {\"1\": \"things\", \"2\": \"things\"}\n state_data = {\n \"job_state\": job_state_data,\n }\n self.mcp.state_watcher.restore.return_value = state_data\n action_runner = mock.Mock()\n self.mcp.restore_state(action_runner)\n self.mcp.jobs.restore_state.assert_called_with(job_state_data, action_runner)\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/mesos_test.py", "content": "from collections import namedtuple\nfrom unittest import mock\n\nimport staticconf.testing\n\nfrom testifycompat import assert_equal\nfrom testifycompat import setup_teardown\nfrom testifycompat import TestCase\nfrom tron.mesos import MesosCluster\nfrom tron.mesos import MesosClusterRepository\nfrom tron.mesos import MesosTask\n\n\nclass TestMesosClusterRepository(TestCase):\n @setup_teardown\n def mock_cluster(self):\n # Ensure different mock is returned each time class is instantiated\n def init_cluster(*args, **kwargs):\n return mock.MagicMock(spec_set=MesosCluster)\n\n with mock.patch(\n \"tron.mesos.MesosCluster\",\n side_effect=init_cluster,\n autospec=True,\n ) as self.cluster_cls:\n yield\n\n def test_get_cluster_repeated_mesos_address(self):\n first = MesosClusterRepository.get_cluster(\"master-a.com\")\n second = MesosClusterRepository.get_cluster(\"master-a.com\")\n assert_equal(first, second)\n assert_equal(self.cluster_cls.call_count, 1)\n\n def test_shutdown(self):\n clusters = [MesosClusterRepository.get_cluster(address) for address in [\"a\", \"b\", \"c\"]]\n assert_equal(self.cluster_cls.call_count, 3)\n MesosClusterRepository.shutdown()\n for cluster in clusters:\n assert_equal(cluster.stop.call_count, 1)\n\n def test_configure(self):\n clusters = [MesosClusterRepository.get_cluster(address) for address in [\"d\", \"e\"]]\n mock_volume = mock.Mock()\n options = mock.Mock(\n master_port=5000,\n secret=\"/dev/null\",\n principal=\"fake-principal\",\n role=\"tron\",\n enabled=False,\n default_volumes=[mock_volume],\n dockercfg_location=\"auth\",\n offer_timeout=1000,\n )\n with mock.patch(\n \"tron.mesos.get_secret_from_file\",\n autospec=True,\n return_value=\"test-secret\",\n ):\n MesosClusterRepository.configure(options)\n\n expected_volume = mock_volume._asdict.return_value\n for cluster in clusters:\n cluster.set_enabled.assert_called_once_with(False)\n cluster.configure_tasks.assert_called_once_with(\n default_volumes=[expected_volume],\n dockercfg_location=\"auth\",\n offer_timeout=1000,\n )\n\n # Next cluster we get should be initialized with the same settings\n MesosClusterRepository.get_cluster(\"f\")\n self.cluster_cls.assert_called_with(\n mesos_address=\"f\",\n mesos_master_port=5000,\n secret=\"test-secret\",\n principal=\"fake-principal\",\n mesos_role=\"tron\",\n framework_id=None,\n enabled=False,\n default_volumes=[expected_volume],\n dockercfg_location=\"auth\",\n offer_timeout=1000,\n )\n\n\ndef mock_task_event(\n task_id,\n platform_type,\n raw=None,\n terminal=False,\n success=False,\n **kwargs,\n):\n return mock.MagicMock(\n kind=\"task\",\n task_id=task_id,\n platform_type=platform_type,\n raw=raw or {},\n terminal=terminal,\n success=success,\n **kwargs,\n )\n\n\nclass TestMesosTask(TestCase):\n @setup_teardown\n def setup(self):\n TaskConfig = namedtuple(\"TaskConfig\", \"cmd task_id cpus mem disk env\")\n self.action_run_id = \"my_service.job.1.action\"\n self.task_id = \"123abcuuid\"\n with mock.patch(\n \"tron.mesos.logging.getLogger\",\n return_value=mock.Mock(handlers=[mock.Mock()]),\n autospec=None,\n ):\n self.task = MesosTask(\n id=self.action_run_id,\n task_config=TaskConfig(\n cmd=\"echo hello world\",\n task_id=self.task_id,\n cpus=0.1,\n mem=100,\n disk=100,\n env={\n \"INITIAL_VAR\": \"baz\",\n \"AWS_SECRET_ACCESS_KEY\": \"THISISASECRET\",\n \"SOME_VAR\": \"bar\",\n \"AWS_ACCESS_KEY_ID\": \"THISISASECRETTOO\",\n \"SOME_OTHER_VAR\": \"foo\",\n },\n ),\n )\n yield\n\n def test_aws_credentials_redacted(self):\n assert all([\"THISISASECRET\" not in text[0][0] for text in self.task.log.info.call_args_list])\n assert all([\"foo\" in text[0][0] for text in self.task.log.info.call_args_list])\n assert all([\"bar\" in text[0][0] for text in self.task.log.info.call_args_list])\n assert all([\"baz\" in text[0][0] for text in self.task.log.info.call_args_list])\n\n def test_handle_staging(self):\n event = mock_task_event(\n task_id=self.task_id,\n platform_type=\"staging\",\n )\n self.task.handle_event(event)\n assert self.task.state == MesosTask.PENDING\n\n def test_handle_starting(self):\n event = mock_task_event(\n task_id=self.task_id,\n platform_type=\"starting\",\n )\n self.task.handle_event(event)\n assert self.task.state == MesosTask.RUNNING\n\n def test_handle_running(self):\n event = mock_task_event(\n task_id=self.task_id,\n platform_type=\"running\",\n )\n self.task.handle_event(event)\n assert self.task.state == MesosTask.RUNNING\n\n def test_handle_running_for_other_task(self):\n event = mock_task_event(\n task_id=\"other321\",\n platform_type=\"running\",\n )\n self.task.handle_event(event)\n assert self.task.state == MesosTask.PENDING\n\n def test_handle_finished(self):\n self.task.started()\n event = mock_task_event(\n task_id=self.task_id,\n platform_type=\"finished\",\n terminal=True,\n success=True,\n )\n self.task.handle_event(event)\n assert self.task.is_complete\n\n def test_handle_failed(self):\n self.task.started()\n event = mock_task_event(\n task_id=self.task_id,\n platform_type=\"failed\",\n terminal=True,\n success=False,\n )\n self.task.handle_event(event)\n assert self.task.is_failed\n assert self.task.is_done\n\n def test_handle_killed(self):\n self.task.started()\n event = mock_task_event(\n task_id=self.task_id,\n platform_type=\"killed\",\n terminal=True,\n success=False,\n )\n self.task.handle_event(event)\n assert self.task.is_failed\n assert self.task.is_done\n\n def test_handle_lost(self):\n self.task.started()\n event = mock_task_event(\n task_id=self.task_id,\n platform_type=\"lost\",\n terminal=True,\n success=False,\n )\n self.task.handle_event(event)\n assert self.task.is_unknown\n assert self.task.is_done\n\n def test_handle_error(self):\n self.task.started()\n event = mock_task_event(\n task_id=self.task_id,\n platform_type=\"error\",\n terminal=True,\n success=False,\n )\n self.task.handle_event(event)\n assert self.task.is_failed\n assert self.task.is_done\n\n def test_handle_terminal_event_offer_timeout(self):\n self.task.started()\n event = mock_task_event(\n task_id=self.task_id,\n platform_type=None,\n terminal=True,\n success=False,\n raw=\"failed due to offer timeout\",\n message=\"stop\",\n )\n self.task.handle_event(event)\n assert self.task.is_failed\n assert self.task.is_done\n\n def test_handle_success_sequence(self):\n self.task.handle_event(\n mock_task_event(\n task_id=self.task_id,\n platform_type=\"staging\",\n ),\n )\n self.task.handle_event(\n mock_task_event(\n task_id=self.task_id,\n platform_type=\"starting\",\n ),\n )\n self.task.handle_event(\n mock_task_event(\n task_id=self.task_id,\n platform_type=\"running\",\n ),\n )\n self.task.handle_event(\n mock_task_event(\n task_id=self.task_id,\n platform_type=\"finished\",\n terminal=True,\n success=True,\n ),\n )\n assert self.task.is_complete\n\n def test_log_event_error(self):\n with mock.patch.object(self.task, \"log_event_info\",) as mock_log_event, mock.patch.object(\n self.task.log,\n \"warning\",\n ) as mock_log:\n mock_log_event.side_effect = Exception\n self.task.handle_event(\n mock_task_event(\n task_id=self.task_id,\n platform_type=\"running\",\n ),\n )\n assert mock_log_event.called\n assert mock_log.called\n assert self.task.state == MesosTask.RUNNING\n\n def test_get_event_logger_add_unique_handlers(self):\n \"\"\"\n Ensures that only a single handler (for stderr) is added to the\n MesosTask event logger, to prevent duplicate log output.\n \"\"\"\n # Call 2 times to make sure 2nd call doesn't add another handler\n logger = self.task.get_event_logger()\n logger = self.task.get_event_logger()\n\n assert len(logger.handlers) == 1\n\n\nclass TestMesosCluster(TestCase):\n @setup_teardown\n def setup_mocks(self):\n with mock.patch(\"tron.mesos.PyDeferredQueue\", autospec=True,) as queue_cls, mock.patch(\n \"tron.mesos.TaskProcessor\",\n autospec=True,\n ) as processor_cls, mock.patch(\"tron.mesos.Subscription\", autospec=True,) as runner_cls, mock.patch(\n \"tron.mesos.get_mesos_leader\",\n autospec=True,\n ) as mock_get_leader:\n self.mock_queue = queue_cls.return_value\n self.mock_processor = processor_cls.return_value\n self.mock_runner_cls = runner_cls\n self.mock_runner_cls.return_value.configure_mock(\n stopping=False,\n TASK_CONFIG_INTERFACE=mock.Mock(),\n )\n self.mock_get_leader = mock_get_leader\n yield\n\n @mock.patch(\"tron.mesos.socket\", autospec=True)\n def test_init(self, mock_socket):\n mock_socket.gethostname.return_value = \"hostname\"\n cluster = MesosCluster(\n mesos_address=\"mesos-cluster-a.me\",\n mesos_master_port=5000,\n secret=\"my_secret\",\n mesos_role=\"tron\",\n framework_id=\"fake_framework_id\",\n principal=\"fake-principal\",\n )\n\n assert_equal(cluster.queue, self.mock_queue)\n assert_equal(cluster.processor, self.mock_processor)\n\n self.mock_get_leader.assert_called_once_with(\n \"mesos-cluster-a.me\",\n 5000,\n )\n self.mock_processor.executor_from_config.assert_has_calls(\n [\n mock.call(\n provider=\"mesos_task\",\n provider_config={\n \"secret\": \"my_secret\",\n \"principal\": \"fake-principal\",\n \"mesos_address\": self.mock_get_leader.return_value,\n \"role\": \"tron\",\n \"framework_name\": \"tron-hostname\",\n \"framework_id\": \"fake_framework_id\",\n \"failover\": True,\n },\n ),\n mock.call(\n provider=\"logging\",\n provider_config=mock.ANY,\n ),\n ]\n )\n self.mock_runner_cls.assert_called_once_with(\n self.mock_processor.executor_from_config.return_value,\n self.mock_queue,\n )\n assert_equal(cluster.runner, self.mock_runner_cls.return_value)\n\n get_event_deferred = cluster.deferred\n assert_equal(get_event_deferred, self.mock_queue.get.return_value)\n get_event_deferred.addCallback.assert_has_calls(\n [\n mock.call(cluster._process_event),\n mock.call(cluster.handle_next_event),\n ]\n )\n\n def test_init_disabled(self):\n cluster = MesosCluster(\"mesos-cluster-a.me\", enabled=False)\n\n assert_equal(cluster.queue, self.mock_queue)\n assert_equal(cluster.processor, self.mock_processor)\n assert_equal(self.mock_processor.executor_from_config.call_count, 0)\n assert cluster.runner is None\n\n def test_set_enabled_off(self):\n cluster = MesosCluster(\"mesos-cluster-a.me\", enabled=True)\n mock_task = mock.Mock()\n cluster.tasks = {\"task\": mock_task}\n cluster.set_enabled(False)\n assert not cluster.enabled\n assert cluster.runner.stop.call_count == 1\n assert cluster.tasks == {}\n assert mock_task.exited.call_count == 1\n\n def test_set_enabled_on(self):\n cluster = MesosCluster(\"mesos-cluster-a.me\", enabled=False)\n cluster.set_enabled(True)\n assert_equal(cluster.enabled, True)\n # Basically the same as regular initialization\n assert_equal(self.mock_processor.executor_from_config.call_count, 2)\n self.mock_runner_cls.assert_called_once_with(\n self.mock_processor.executor_from_config.return_value,\n self.mock_queue,\n )\n assert_equal(cluster.runner, self.mock_runner_cls.return_value)\n\n get_event_deferred = cluster.deferred\n assert_equal(get_event_deferred, self.mock_queue.get.return_value)\n get_event_deferred.addCallback.assert_has_calls(\n [\n mock.call(cluster._process_event),\n mock.call(cluster.handle_next_event),\n ]\n )\n\n def test_set_enabled_on_already(self):\n cluster = MesosCluster(\"mesos-cluster-a.me\", enabled=True)\n cluster.set_enabled(True)\n assert_equal(cluster.enabled, True)\n # Runner should have only be created once\n assert_equal(self.mock_runner_cls.call_count, 1)\n\n def test_configure_tasks(self):\n cluster = MesosCluster(\n \"mesos-cluster-a.me\",\n default_volumes=[],\n dockercfg_location=\"first\",\n offer_timeout=60,\n )\n assert_equal(cluster.default_volumes, [])\n assert_equal(cluster.dockercfg_location, \"first\")\n assert_equal(cluster.offer_timeout, 60)\n\n expected_volumes = [\n {\n \"container_path\": \"/tmp\",\n \"host_path\": \"/host\",\n \"mode\": \"RO\",\n }\n ]\n cluster.configure_tasks(\n default_volumes=expected_volumes,\n dockercfg_location=\"second\",\n offer_timeout=300,\n )\n assert_equal(cluster.default_volumes, expected_volumes)\n assert_equal(cluster.dockercfg_location, \"second\")\n assert_equal(cluster.offer_timeout, 300)\n\n def test_submit(self):\n mock_clusterman_metrics = mock.MagicMock()\n cluster = MesosCluster(\"mesos-cluster-a.me\")\n mock_task = mock.MagicMock(get_config=mock.Mock(return_value={\"environment\": {}}))\n mock_task.get_mesos_id.return_value = \"this_task\"\n with mock.patch(\n \"tron.mesos.get_clusterman_metrics\",\n return_value=(mock_clusterman_metrics),\n autospec=True,\n ):\n cluster.submit(mock_task)\n\n assert \"this_task\" in cluster.tasks\n assert cluster.tasks[\"this_task\"] == mock_task\n cluster.runner.run.assert_called_once_with(\n mock_task.get_config.return_value,\n )\n assert mock_clusterman_metrics.ClustermanMetricsBotoClient.call_count == 0\n\n def test_submit_with_clusterman(self):\n mock_clusterman_metrics = mock.MagicMock()\n cluster = MesosCluster(\"mesos-cluster-a.me\")\n mock_task = mock.MagicMock(\n get_config=mock.Mock(\n return_value={\n \"environment\": {\n \"CLUSTERMAN_RESOURCES\": '{\"required_cpus|blah=x\": 4}',\n \"EXECUTOR_CLUSTER\": \"fake-cluster\",\n \"EXECUTOR_POOL\": \"fake-pool\",\n },\n },\n ),\n )\n mock_task.get_mesos_id.return_value = \"this_task\"\n with mock.patch(\n \"tron.mesos.get_clusterman_metrics\",\n return_value=mock_clusterman_metrics,\n autospec=True,\n ), staticconf.testing.MockConfiguration(\n {\"clusters\": {\"fake-cluster\": {\"aws_region\": \"fake-region\"}}},\n namespace=\"clusterman\",\n ):\n cluster.submit(mock_task)\n\n assert \"this_task\" in cluster.tasks\n assert cluster.tasks[\"this_task\"] == mock_task\n cluster.runner.run.assert_called_once_with(\n mock_task.get_config.return_value,\n )\n assert mock_clusterman_metrics.ClustermanMetricsBotoClient.call_count == 1\n\n def test_submit_disabled(self):\n cluster = MesosCluster(\"mesos-cluster-a.me\", enabled=False)\n mock_task = mock.MagicMock()\n mock_task.get_mesos_id.return_value = \"this_task\"\n with mock.patch(\n \"tron.mesos.get_clusterman_metrics\",\n return_value=(None, None),\n autospec=True,\n ):\n cluster.submit(mock_task)\n\n assert \"this_task\" not in cluster.tasks\n mock_task.exited.assert_called_once_with(1)\n\n def test_recover(self):\n cluster = MesosCluster(\"mesos-cluster-a.me\")\n mock_task = mock.MagicMock()\n mock_task.get_mesos_id.return_value = \"this_task\"\n cluster.recover(mock_task)\n\n assert \"this_task\" in cluster.tasks\n assert cluster.tasks[\"this_task\"] == mock_task\n cluster.runner.reconcile.assert_called_once_with(\n mock_task.get_config.return_value,\n )\n assert mock_task.started.call_count == 1\n\n def test_recover_disabled(self):\n cluster = MesosCluster(\"mesos-cluster-a.me\", enabled=False)\n mock_task = mock.MagicMock()\n mock_task.get_mesos_id.return_value = \"this_task\"\n cluster.recover(mock_task)\n\n assert \"this_task\" not in cluster.tasks\n mock_task.exited.assert_called_once_with(None)\n\n @mock.patch(\"tron.mesos.MesosTask\", autospec=True)\n def test_create_task_defaults(self, mock_task):\n cluster = MesosCluster(\"mesos-cluster-a.me\")\n mock_serializer = mock.MagicMock()\n task = cluster.create_task(\n action_run_id=\"action_c\",\n command=\"echo hi\",\n cpus=1,\n mem=10,\n disk=20,\n constraints=[],\n docker_image=\"container:latest\",\n docker_parameters=[],\n env={\"TESTING\": \"true\"},\n extra_volumes=[],\n serializer=mock_serializer,\n )\n cluster.runner.TASK_CONFIG_INTERFACE.assert_called_once_with(\n name=\"action_c\",\n cmd=\"echo hi\",\n cpus=1,\n mem=10,\n disk=20,\n constraints=[],\n image=\"container:latest\",\n docker_parameters=[],\n environment={\"TESTING\": \"true\"},\n volumes=[],\n uris=[],\n offer_timeout=None,\n )\n assert_equal(task, mock_task.return_value)\n mock_task.assert_called_once_with(\n \"action_c\",\n cluster.runner.TASK_CONFIG_INTERFACE.return_value,\n mock_serializer,\n )\n\n @mock.patch(\"tron.mesos.MesosTask\", autospec=True)\n def test_create_task_with_task_id(self, mock_task):\n cluster = MesosCluster(\"mesos-cluster-a.me\")\n mock_serializer = mock.MagicMock()\n task_id = \"task.0123-fabc\"\n task = cluster.create_task(\n action_run_id=\"action_c\",\n command=\"echo hi\",\n cpus=1,\n mem=10,\n disk=20,\n constraints=[],\n docker_image=\"container:latest\",\n docker_parameters=[],\n env={\"TESTING\": \"true\"},\n extra_volumes=[],\n serializer=mock_serializer,\n task_id=task_id,\n )\n assert cluster.runner.TASK_CONFIG_INTERFACE.call_count == 1\n assert task == mock_task.return_value\n task_config = cluster.runner.TASK_CONFIG_INTERFACE.return_value\n task_config.set_task_id.assert_called_once_with(task_id)\n mock_task.assert_called_once_with(\n \"action_c\",\n task_config.set_task_id.return_value,\n mock_serializer,\n )\n\n @mock.patch(\"tron.mesos.MesosTask\", autospec=True)\n def test_create_task_disabled(self, mock_task):\n # If Mesos is disabled, should return None\n cluster = MesosCluster(\"mesos-cluster-a.me\", enabled=False)\n mock_serializer = mock.MagicMock()\n task = cluster.create_task(\n action_run_id=\"action_c\",\n command=\"echo hi\",\n cpus=1,\n mem=10,\n disk=20,\n constraints=[],\n docker_image=\"container:latest\",\n docker_parameters=[],\n env={\"TESTING\": \"true\"},\n extra_volumes=[],\n serializer=mock_serializer,\n )\n assert task is None\n\n @mock.patch(\"tron.mesos.MesosTask\", autospec=True)\n def test_create_task_with_configuration(self, mock_task):\n cluster = MesosCluster(\n \"mesos-cluster-a.me\",\n default_volumes=[\n {\n \"container_path\": \"/tmp\",\n \"host_path\": \"/host\",\n \"mode\": \"RO\",\n },\n {\n \"container_path\": \"/other\",\n \"host_path\": \"/other\",\n \"mode\": \"RW\",\n },\n ],\n dockercfg_location=\"some_place\",\n offer_timeout=202,\n )\n mock_serializer = mock.MagicMock()\n task = cluster.create_task(\n action_run_id=\"action_c\",\n command=\"echo hi\",\n cpus=1,\n mem=10,\n disk=20,\n constraints=[],\n docker_image=\"container:latest\",\n docker_parameters=[],\n env={\"TESTING\": \"true\"},\n # This should override the default volume for /tmp\n extra_volumes=[\n {\n \"container_path\": \"/tmp\",\n \"host_path\": \"/custom\",\n \"mode\": \"RW\",\n },\n ],\n serializer=mock_serializer,\n )\n cluster.runner.TASK_CONFIG_INTERFACE.assert_called_once_with(\n name=\"action_c\",\n cmd=\"echo hi\",\n cpus=1,\n mem=10,\n disk=20,\n constraints=[],\n image=\"container:latest\",\n docker_parameters=[],\n environment={\"TESTING\": \"true\"},\n volumes=[\n {\n \"container_path\": \"/tmp\",\n \"host_path\": \"/custom\",\n \"mode\": \"RW\",\n },\n {\n \"container_path\": \"/other\",\n \"host_path\": \"/other\",\n \"mode\": \"RW\",\n },\n ],\n uris=[\"some_place\"],\n offer_timeout=202,\n )\n assert_equal(task, mock_task.return_value)\n mock_task.assert_called_once_with(\n \"action_c\",\n cluster.runner.TASK_CONFIG_INTERFACE.return_value,\n mock_serializer,\n )\n\n def test_process_event_task(self):\n event = mock_task_event(\"this_task\", \"some_platform_type\")\n cluster = MesosCluster(\"mesos-cluster-a.me\")\n mock_task = mock.MagicMock(spec_set=MesosTask)\n mock_task.get_mesos_id.return_value = \"this_task\"\n cluster.tasks[\"this_task\"] = mock_task\n\n cluster._process_event(event)\n mock_task.handle_event.assert_called_once_with(event)\n\n def test_process_event_task_id_invalid(self):\n event = mock_task_event(\"other_task\", \"some_platform_type\")\n cluster = MesosCluster(\"mesos-cluster-a.me\")\n mock_task = mock.MagicMock(spec_set=MesosTask)\n mock_task.get_mesos_id.return_value = \"this_task\"\n cluster.tasks[\"this_task\"] = mock_task\n\n cluster._process_event(event)\n assert_equal(mock_task.handle_event.call_count, 0)\n\n def test_process_event_control_stop(self):\n event = mock.MagicMock(\n kind=\"control\",\n message=\"stop\",\n )\n cluster = MesosCluster(\"mesos-cluster-a.me\")\n cluster._process_event(event)\n assert cluster.runner.stop.call_count == 1\n assert cluster.deferred is None\n\n def test_stop_default(self):\n # When stopping, tasks should not exit. They will be recovered\n cluster = MesosCluster(\"mesos-cluster-a.me\")\n mock_task = mock.MagicMock()\n cluster.tasks = {\"task_id\": mock_task}\n cluster.stop()\n assert cluster.runner.stop.call_count == 1\n assert cluster.deferred is None\n assert mock_task.exited.call_count == 0\n assert len(cluster.tasks) == 1\n\n def test_stop_disabled(self):\n # Shouldn't raise an error\n cluster = MesosCluster(\"mesos-cluster-a.me\", enabled=False)\n cluster.stop()\n\n def test_kill(self):\n cluster = MesosCluster(\"mesos-cluster-a.me\")\n cluster.kill(\"fake_task_id\")\n cluster.runner.kill.assert_called_once_with(\"fake_task_id\")\n" }, { "path": "tests/metrics_test.py", "content": "from unittest import mock\n\nimport pytest\n\nimport tron.metrics as metrics\n\n\n@pytest.fixture(autouse=True)\ndef all_metrics():\n with mock.patch.object(metrics, \"all_metrics\", new=dict()) as mock_all:\n yield mock_all\n\n\ndef test_get_metric(all_metrics):\n timer = metrics.get_metric(\n \"timer\",\n \"api.requests\",\n {\"method\": \"GET\"},\n mock.Mock(),\n )\n same_timer = metrics.get_metric(\n \"timer\",\n \"api.requests\",\n {\"method\": \"GET\"},\n mock.Mock(),\n )\n other_timer = metrics.get_metric(\n \"timer\",\n \"api.requests\",\n {\"method\": \"POST\"},\n mock.Mock(),\n )\n metrics.get_metric(\"something\", \"name\", None, mock.Mock())\n assert timer == same_timer\n assert other_timer != timer\n assert len(all_metrics) == 3\n\n\n@mock.patch(\"tron.metrics.get_metric\", autospec=True)\ndef test_timer(mock_get_metric):\n test_metric = metrics.Timer()\n mock_get_metric.return_value = test_metric\n metrics.timer(\"my_metric\", 110)\n metrics.timer(\"my_metric\", 84)\n mock_get_metric.assert_called_with(\n \"timer\",\n \"my_metric\",\n None,\n mock.ANY,\n )\n result = metrics.view_timer(test_metric)\n assert result[\"count\"] == 2\n\n\n@mock.patch(\"tron.metrics.get_metric\", autospec=True)\ndef test_count(mock_get_metric):\n test_metric = metrics.Counter()\n mock_get_metric.return_value = test_metric\n metrics.count(\"my_metric\", 13)\n metrics.count(\"my_metric\", -1)\n mock_get_metric.assert_called_with(\n \"counter\",\n \"my_metric\",\n None,\n mock.ANY,\n )\n result = metrics.view_counter(test_metric)\n assert result[\"count\"] == 12\n\n\n@mock.patch(\"tron.metrics.get_metric\", autospec=True)\ndef test_meter(mock_get_metric):\n test_metric = metrics.Meter()\n mock_get_metric.return_value = test_metric\n metrics.meter(\"my_metric\")\n metrics.meter(\"my_metric\")\n mock_get_metric.assert_called_with(\n \"meter\",\n \"my_metric\",\n None,\n mock.ANY,\n )\n result = metrics.view_meter(test_metric)\n assert result[\"count\"] == 2\n\n\n@mock.patch(\"tron.metrics.get_metric\", autospec=True)\ndef test_gauge(mock_get_metric):\n test_metric = metrics.SimpleGauge()\n mock_get_metric.return_value = test_metric\n metrics.gauge(\"my_metric\", 23)\n metrics.gauge(\"my_metric\", 102)\n mock_get_metric.assert_called_with(\n \"gauge\",\n \"my_metric\",\n None,\n mock.ANY,\n )\n result = metrics.view_gauge(test_metric)\n assert result[\"value\"] == 102\n\n\n@mock.patch(\"tron.metrics.get_metric\", autospec=True)\ndef test_histogram(mock_get_metric):\n test_metric = metrics.Histogram()\n mock_get_metric.return_value = test_metric\n metrics.histogram(\"my_metric\", 2)\n metrics.histogram(\"my_metric\", 92)\n mock_get_metric.assert_called_with(\n \"histogram\",\n \"my_metric\",\n None,\n mock.ANY,\n )\n result = metrics.view_histogram(test_metric)\n assert result[\"count\"] == 2\n\n\ndef test_view_all_metrics_empty():\n result = metrics.view_all_metrics()\n assert result == {\n \"counter\": [],\n \"gauge\": [],\n \"histogram\": [],\n \"meter\": [],\n \"timer\": [],\n }\n\n\ndef test_view_all_metrics():\n metrics.timer(\"a\", 1)\n metrics.count(\"b\", 9, dimensions={\"method\": \"GET\"})\n metrics.meter(\"c\")\n metrics.gauge(\"d\", 3)\n metrics.histogram(\"e\", 2)\n metrics.histogram(\"f\", 3)\n result = metrics.view_all_metrics()\n\n assert len(result[\"timer\"]) == 1\n assert result[\"timer\"][0][\"name\"] == \"a\"\n\n assert len(result[\"counter\"]) == 1\n assert result[\"counter\"][0][\"name\"] == \"b\"\n assert result[\"counter\"][0][\"dimensions\"] == {\"method\": \"GET\"}\n\n assert len(result[\"meter\"]) == 1\n assert result[\"meter\"][0][\"name\"] == \"c\"\n\n assert len(result[\"gauge\"]) == 1\n assert result[\"gauge\"][0][\"name\"] == \"d\"\n\n assert len(result[\"histogram\"]) == 2\n names = {metric[\"name\"] for metric in result[\"histogram\"]}\n assert names == {\"e\", \"f\"}\n" }, { "path": "tests/mocks.py", "content": "import atexit\nimport datetime\nimport itertools\nimport shutil\nimport tempfile\nfrom unittest.mock import MagicMock\n\n\nclass MockAction(MagicMock):\n def __init__(self, *args, **kwargs):\n kwargs.setdefault(\"name\", \"action_name\")\n kwargs.setdefault(\"required_actions\", [])\n kwargs.setdefault(\"dependent_actions\", [])\n super().__init__(*args, **kwargs)\n\n\nclass MockActionGraph(MagicMock):\n def __init__(self, *args, **kwargs):\n action = MockAction()\n kwargs.setdefault(\"graph\", [action])\n kwargs.setdefault(\"action_map\", {action.name: action})\n super().__init__(*args, **kwargs)\n\n def __getitem__(self, item):\n action = MockAction(name=item)\n self.action_map.setdefault(item, action)\n return self.action_map[item]\n\n def get_required_actions(self, name):\n return []\n\n\nclass MockActionRun(MagicMock):\n def __init__(self, *args, **kwargs):\n kwargs.setdefault(\"output_path\", [tempfile.mkdtemp()])\n kwargs.setdefault(\"start_time\", datetime.datetime.now())\n kwargs.setdefault(\"end_time\", datetime.datetime.now())\n atexit.register(lambda: shutil.rmtree(kwargs[\"output_path\"][0]))\n super().__init__(*args, **kwargs)\n\n\nclass MockActionRunCollection(MagicMock):\n def __init__(self, *args, **kwargs):\n kwargs.setdefault(\"action_graph\", MockActionGraph())\n kwargs.setdefault(\"run_map\", {})\n super().__init__(*args, **kwargs)\n\n def __getitem__(self, item):\n action_run = MockActionRun(name=item)\n self.run_map.setdefault(item, action_run)\n return self.run_map[item]\n\n\nclass MockJobRun(MagicMock):\n def __init__(self, *args, **kwargs):\n kwargs.setdefault(\"output_path\", [tempfile.mkdtemp()])\n kwargs.setdefault(\"action_graph\", MockActionGraph())\n action_runs = MockActionRunCollection(\n action_graph=kwargs[\"action_graph\"],\n )\n kwargs.setdefault(\"action_runs\", action_runs)\n atexit.register(lambda: shutil.rmtree(kwargs[\"output_path\"][0]))\n super().__init__(*args, **kwargs)\n\n\nclass MockNode(MagicMock):\n def __init__(self, hostname=None):\n super().__init__()\n self.name = self.hostname = hostname\n\n def run(self, runnable):\n runnable.started()\n return type(self)()\n\n\nclass MockNodePool:\n _node = None\n\n def __init__(self, *node_names):\n self.nodes = []\n self._ndx_cycle = None\n for hostname in node_names:\n self.nodes.append(MockNode(hostname=hostname))\n\n if self.nodes:\n self._ndx_cycle = itertools.cycle(range(0, len(self.nodes)))\n\n def __getitem__(self, value):\n for node in self.nodes:\n if node.hostname == value:\n return node\n else:\n raise KeyError\n\n def next(self):\n if not self.nodes:\n self.nodes.append(MockNode())\n\n if self._ndx_cycle:\n return self.nodes[next(self._ndx_cycle)]\n else:\n return self.nodes[0]\n\n next_round_robin = next\n\n\nclass MockJobRunCollection(MagicMock):\n def __iter__(self):\n return iter(self.runs)\n" }, { "path": "tests/node_test.py", "content": "from unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_in\nfrom testifycompat import assert_not_equal\nfrom testifycompat import assert_not_in\nfrom testifycompat import assert_raises\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import setup_teardown\nfrom testifycompat import teardown\nfrom testifycompat import TestCase\nfrom tests.testingutils import autospec_method\nfrom tron import actioncommand\nfrom tron import node\nfrom tron import ssh\nfrom tron.config import schema\nfrom tron.core import actionrun\nfrom tron.serialize import filehandler\n\n\ndef create_mock_node(name=None):\n mock_node = mock.create_autospec(node.Node)\n if name:\n mock_node.get_name.return_value = name\n return mock_node\n\n\ndef create_mock_pool():\n return mock.create_autospec(node.NodePool)\n\n\nclass TestNodePoolRepository(TestCase):\n @setup\n def setup_store(self):\n self.node = create_mock_node()\n self.repo = node.NodePoolRepository.get_instance()\n self.repo.add_node(self.node)\n\n @teardown\n def teardown_store(self):\n self.repo.clear()\n\n def test_single_instance(self):\n assert_raises(ValueError, node.NodePoolRepository)\n assert self.repo is node.NodePoolRepository.get_instance()\n\n def test_get_by_name(self):\n node_pool = self.repo.get_by_name(self.node.get_name())\n assert_equal(self.node, node_pool.next())\n\n def test_get_by_name_miss(self):\n assert_equal(None, self.repo.get_by_name(\"bogus\"))\n\n def test_clear(self):\n self.repo.clear()\n assert_not_in(self.node, self.repo.nodes)\n assert_not_in(self.node, self.repo.pools)\n\n def test_update_from_config(self):\n mock_nodes = {\"a\": create_mock_node(\"a\"), \"b\": create_mock_node(\"b\")}\n self.repo.nodes.update(mock_nodes)\n node_config = {\"a\": mock.Mock(), \"b\": mock.Mock()}\n node_pool_config = {\"c\": mock.Mock(nodes=[\"a\", \"b\"])}\n ssh_options = mock.Mock(identities=[], known_hosts_file=None)\n node.NodePoolRepository.update_from_config(\n node_config,\n node_pool_config,\n ssh_options,\n )\n node_names = [node_config[\"a\"].name, node_config[\"b\"].name]\n assert_equal(\n set(self.repo.pools),\n set(node_names + [node_pool_config[\"c\"].name]),\n )\n assert_equal(\n set(self.repo.nodes),\n set(list(node_names) + list(mock_nodes.keys())),\n )\n\n def test_nodes_by_name(self):\n mock_nodes = {\"a\": mock.Mock(), \"b\": mock.Mock()}\n self.repo.nodes.update(mock_nodes)\n nodes = self.repo._get_nodes_by_name([\"a\", \"b\"])\n assert_equal(nodes, list(mock_nodes.values()))\n\n def test_get_node(self):\n returned_node = self.repo.get_node(self.node.get_name())\n assert_equal(returned_node, self.node)\n\n\nclass TestKnownHost(TestCase):\n @setup\n def setup_known_hosts(self):\n self.known_hosts = node.KnownHosts(None)\n self.entry = mock.Mock()\n self.known_hosts._added.append(self.entry)\n\n def test_get_public_key(self):\n hostname = \"hostname\"\n pub_key = self.known_hosts.get_public_key(hostname)\n self.entry.matchesHost.assert_called_with(hostname)\n assert_equal(pub_key, self.entry.publicKey)\n\n def test_get_public_key_not_found(self):\n self.entry.matchesHost.return_value = False\n assert not self.known_hosts.get_public_key(\"hostname\")\n\n\nclass TestDetermineJitter(TestCase):\n @setup\n def setup_node_settings(self):\n self.settings = mock.Mock(\n jitter_load_factor=1,\n jitter_min_load=4,\n jitter_max_delay=20,\n )\n\n @setup_teardown\n def patch_random(self):\n with mock.patch(\"tron.node.random\", autospec=True) as mock_random:\n mock_random.random.return_value = 1\n yield\n\n def test_jitter_under_min_load(self):\n assert_equal(node.determine_jitter(3, self.settings), 0)\n assert_equal(node.determine_jitter(4, self.settings), 0)\n\n def test_jitter_with_load_factor(self):\n self.settings.jitter_load_factor = 2\n assert_equal(node.determine_jitter(3, self.settings), 2.0)\n assert_equal(node.determine_jitter(2, self.settings), 0)\n\n def test_jitter_with_max_delay(self):\n self.settings.jitter_max_delay = 15\n assert_equal(node.determine_jitter(20, self.settings), 15.0)\n assert_equal(node.determine_jitter(100, self.settings), 15.0)\n\n\ndef build_node(\n hostname=\"localhost\",\n username=\"theuser\",\n name=\"thename\",\n pub_key=None,\n):\n config = mock.Mock(hostname=hostname, username=username, name=name)\n ssh_opts = mock.create_autospec(ssh.SSHAuthOptions)\n node_settings = mock.create_autospec(schema.ConfigSSHOptions)\n return node.Node(config, ssh_opts, pub_key, node_settings)\n\n\nclass TestNode(TestCase):\n class TestConnection:\n def openChannel(self, chan):\n self.chan = chan\n\n @setup\n def setup_node(self):\n self.node = build_node()\n\n def test_output_logging(self):\n test_node = build_node()\n serializer = mock.create_autospec(filehandler.FileHandleManager)\n action_cmd = actionrun.ActionCommand(\"test\", \"false\", serializer)\n\n test_node.connection = self.TestConnection()\n test_node.run_states = {action_cmd.id: mock.Mock(state=0)}\n test_node.run_states[action_cmd.id].state = node.RUN_STATE_CONNECTING\n test_node.run_states[action_cmd.id].run = action_cmd\n\n test_node._open_channel(action_cmd)\n assert test_node.connection.chan is not None\n test_node.connection.chan.dataReceived(\"test\")\n serializer.open.return_value.write.assert_called_with(\"test\")\n\n def test_from_config(self):\n ssh_options = self.node.conch_options\n node_config = mock.Mock(\n hostname=\"localhost\",\n username=\"theuser\",\n name=\"thename\",\n )\n ssh_options.__getitem__.return_value = \"something\"\n public_key = mock.Mock()\n node_settings = mock.Mock()\n new_node = node.Node.from_config(\n node_config,\n ssh_options,\n public_key,\n node_settings,\n )\n assert_equal(new_node.name, node_config.name)\n assert_equal(new_node.hostname, node_config.hostname)\n assert_equal(new_node.username, node_config.username)\n assert_equal(new_node.pub_key, public_key)\n assert_equal(new_node.node_settings, node_settings)\n\n def test__eq__true(self):\n other_node = build_node()\n other_node.conch_options = self.node.conch_options\n other_node.node_settings = self.node.node_settings\n other_node.config = self.node.config\n assert_equal(other_node, self.node)\n\n def test__eq__false_config_changed(self):\n other_node = build_node(username=\"different\")\n assert_not_equal(other_node, self.node)\n\n def test__eq__false_pub_key_changed(self):\n other_node = build_node(pub_key=\"something\")\n assert_not_equal(other_node, self.node)\n\n def test__eq__false_ssh_options_changed(self):\n other_node = build_node()\n other_node.conch_options = mock.create_autospec(ssh.SSHAuthOptions)\n assert_not_equal(other_node, self.node)\n\n def test_stop_not_tracked(self):\n action_command = mock.create_autospec(\n actioncommand.ActionCommand,\n id=mock.Mock(),\n )\n self.node.stop(action_command)\n\n def test_stop(self):\n autospec_method(self.node._fail_run)\n action_command = mock.create_autospec(\n actioncommand.ActionCommand,\n id=mock.Mock(),\n )\n self.node.run_states[action_command.id] = mock.Mock()\n self.node.stop(action_command)\n assert_equal(self.node._fail_run.call_count, 1)\n\n\nclass TestNodePool(TestCase):\n @setup\n def setup_nodes(self):\n self.nodes = [build_node(name=\"node%s\" % i) for i in range(5)]\n self.node_pool = node.NodePool(self.nodes, \"thename\")\n\n def test_from_config(self):\n name = \"the pool name\"\n nodes = [create_mock_node(), create_mock_node()]\n config = mock.Mock(name=name)\n new_pool = node.NodePool.from_config(config, nodes)\n assert_equal(new_pool.name, config.name)\n assert_equal(new_pool.nodes, nodes)\n\n def test__init__(self):\n new_node = node.NodePool(self.nodes, \"thename\")\n assert_equal(new_node.name, \"thename\")\n\n def test__eq__(self):\n other_pool = node.NodePool(self.nodes, \"othername\")\n assert_equal(self.node_pool, other_pool)\n\n def test_next(self):\n # Call next many times\n for _ in range(len(self.nodes) * 2 + 1):\n assert_in(self.node_pool.next(), self.nodes)\n\n def test_next_round_robin(self):\n node_order = [self.node_pool.next_round_robin() for _ in range(len(self.nodes) * 2)]\n assert_equal(node_order, self.nodes + self.nodes)\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/sandbox.py", "content": "import contextlib\nimport functools\nimport logging\nimport os\nimport shutil\nimport signal\nimport socket\nimport sys\nimport tempfile\nimport time\nfrom subprocess import CalledProcessError\nfrom subprocess import PIPE\nfrom subprocess import Popen\nfrom unittest import mock\n\nfrom testifycompat import assert_not_equal\nfrom testifycompat import setup\nfrom testifycompat import teardown\nfrom testifycompat import TestCase\nfrom tron.commands import client\nfrom tron.config import manager\nfrom tron.config import schema\n\n# Used for getting the locations of the executable\ntest_dir, _ = os.path.split(__file__)\nrepo_root, _ = os.path.split(test_dir)\n\nlog = logging.getLogger(__name__)\n\n\ndef wait_on_sandbox(func, delay=0.1, max_wait=5.0):\n \"\"\"Poll for func() to return True. Sleeps `delay` seconds between polls\n up to a max of `max_wait` seconds.\n \"\"\"\n start_time = time.time()\n while time.time() - start_time < max_wait:\n time.sleep(delay)\n if func():\n return\n raise TronSandboxException(\"Failed %s\" % func.__name__)\n\n\ndef wait_on_state(client_func, url, state, field=\"state\"):\n \"\"\"Use client_func(url) to wait until the resource changes to state.\"\"\"\n\n def wait_func():\n return client_func(url)[field] == state\n\n wait_func.__name__ = f\"{url} wait on {state}\"\n wait_on_sandbox(wait_func)\n\n\ndef wait_on_proc_terminate(pid):\n def wait_on_terminate():\n try:\n os.kill(pid, 0)\n except Exception:\n return True\n\n wait_on_terminate.__name__ = \"Wait on %s to terminate\" % pid\n wait_on_sandbox(wait_on_terminate)\n\n\ndef build_waiter_func(client_func, url):\n return functools.partial(wait_on_state, client_func, url)\n\n\ndef handle_output(cmd, out_err, returncode):\n \"\"\"Log process output before it is parsed. Raise exception if exit code\n is nonzero.\n \"\"\"\n stdout, stderr = out_err\n cmd = \" \".join(cmd)\n if stdout:\n log.warn(\"%s STDOUT: %s\", cmd, stdout)\n if stderr:\n log.warn(\"%s STDERR: %s\", cmd, stderr)\n if returncode:\n raise CalledProcessError(returncode, cmd)\n\n\ndef find_unused_port():\n \"\"\"Return a port number that is not in use.\"\"\"\n sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n with contextlib.closing(sock) as sock:\n sock.bind((\"localhost\", 0))\n _, port = sock.getsockname()\n return port\n\n\nclass TronSandboxException(Exception):\n pass\n\n\nclass SandboxTestCase(TestCase):\n\n _suites = [\"sandbox\"]\n\n sandbox = None\n\n @setup\n def make_sandbox(self):\n verify_environment()\n self.sandbox = TronSandbox()\n self.client = self.sandbox.client\n\n @teardown\n def delete_sandbox(self):\n if self.sandbox:\n self.sandbox.delete()\n self.sandbox = None\n\n def start_with_config(self, config):\n self.sandbox.save_config(config)\n self.sandbox.trond()\n\n def restart_trond(self):\n old_pid = self.sandbox.get_trond_pid()\n self.sandbox.shutdown_trond()\n wait_on_proc_terminate(self.sandbox.get_trond_pid())\n\n self.sandbox.trond()\n assert_not_equal(old_pid, self.sandbox.get_trond_pid())\n\n\nclass ClientProxy:\n \"\"\"Wrap calls to client and raise a TronSandboxException on connection\n failures.\n \"\"\"\n\n def __init__(self, client, log_filename):\n self.client = client\n self.log_filename = log_filename\n\n def log_contents(self):\n \"\"\"Return the contents of the log file.\"\"\"\n with open(self.log_filename) as f:\n return f.read()\n\n def wrap(self, func, *args, **kwargs):\n with mock.patch(\"tron.commands.client.log\", autospec=True):\n try:\n return func(*args, **kwargs)\n except (client.RequestError, ValueError) as e:\n # ValueError for JSONDecode errors\n log_contents = self.log_contents()\n if log_contents:\n log.warn(f\"{e!r}, Log:\\n{log_contents}\")\n return False\n\n def __getattr__(self, name):\n attr = getattr(self.client, name)\n if not callable(attr):\n return attr\n\n return functools.partial(self.wrap, attr)\n\n\ndef verify_environment():\n for env_var in [\"SSH_AUTH_SOCK\", \"PYTHONPATH\"]:\n if not os.environ.get(env_var):\n raise TronSandboxException(\n \"Missing $%s in test environment.\" % env_var,\n )\n\n\nclass TronSandbox:\n \"\"\"A sandbox for running trond and tron commands in subprocesses.\"\"\"\n\n def __init__(self):\n \"\"\"Set up a temp directory and store paths to relevant binaries\"\"\"\n self.tmp_dir = tempfile.mkdtemp(prefix=\"tron-\")\n cmd_path_func = functools.partial(os.path.join, repo_root, \"bin\")\n cmds = \"tronctl\", \"trond\", \"tronfig\", \"tronview\"\n self.commands = {cmd: cmd_path_func(cmd) for cmd in cmds}\n self.log_file = self.abs_path(\"tron.log\")\n self.log_conf = self.abs_path(\"logging.conf\")\n self.pid_file = self.abs_path(\"tron.pid\")\n self.config_path = self.abs_path(\"configs/\")\n self.port = find_unused_port()\n self.host = \"localhost\"\n self.api_uri = f\"http://{self.host}:{self.port}\"\n cclient = client.Client(self.api_uri)\n self.client = ClientProxy(cclient, self.log_file)\n self.setup_logging_conf()\n\n def abs_path(self, filename):\n \"\"\"Return the absolute path for a file in the sandbox.\"\"\"\n return os.path.join(self.tmp_dir, filename)\n\n def setup_logging_conf(self):\n config_template = os.path.join(repo_root, \"tests/data/logging.conf\")\n with open(config_template) as fh:\n config = fh.read()\n\n with open(self.log_conf, \"w\") as fh:\n fh.write(config.format(self.log_file))\n\n def delete(self):\n \"\"\"Delete the temp directory and shutdown trond.\"\"\"\n self.shutdown_trond(sig_num=signal.SIGKILL)\n shutil.rmtree(self.tmp_dir)\n\n def save_config(self, config_text):\n \"\"\"Save the initial tron configuration.\"\"\"\n manager.create_new_config(self.config_path, config_text)\n\n def run_command(self, command_name, args=None, stdin_lines=None):\n \"\"\"Run the command by name and return (stdout, stderr).\"\"\"\n args = args or []\n command = [sys.executable, self.commands[command_name]] + args\n stdin = PIPE if stdin_lines else None\n proc = Popen(command, stdout=PIPE, stderr=PIPE, stdin=stdin)\n streams = proc.communicate(stdin_lines)\n try:\n handle_output(command, streams, proc.returncode)\n except CalledProcessError:\n log.warn(self.client.log_contents())\n raise\n return streams\n\n def tronctl(self, *args):\n args = list(args) if args else []\n return self.run_command(\"tronctl\", args + [\"--server\", self.api_uri])\n\n def tronview(self, *args):\n args = list(args) if args else []\n args += [\"--nocolor\", \"--server\", self.api_uri]\n return self.run_command(\"tronview\", args)\n\n def trond(self, *args):\n args = list(args) if args else []\n args += [\n \"--working-dir=%s\" % self.tmp_dir,\n \"--pid-file=%s\" % self.pid_file,\n \"--port=%d\" % self.port,\n \"--host=%s\" % self.host,\n \"--config-path=%s\" % self.config_path,\n \"--log-conf=%s\" % self.log_conf,\n ]\n\n self.run_command(\"trond\", args)\n wait_on_sandbox(lambda: bool(self.client.home()))\n\n def tronfig(\n self,\n config_content=None,\n name=schema.MASTER_NAMESPACE,\n ):\n args = [\"--server\", self.api_uri, name]\n args += [\"-\"] if config_content else [\"-p\"]\n return self.run_command(\"tronfig\", args, stdin_lines=config_content)\n\n def get_trond_pid(self):\n if not os.path.exists(self.pid_file):\n return None\n with open(self.pid_file) as f:\n return int(f.read())\n\n def shutdown_trond(self, sig_num=signal.SIGTERM):\n trond_pid = self.get_trond_pid()\n if trond_pid:\n os.kill(trond_pid, sig_num)\n" }, { "path": "tests/scheduler_test.py", "content": "import calendar\nimport datetime\nfrom unittest import mock\n\nimport pytz\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_gt\nfrom testifycompat import assert_gte\nfrom testifycompat import assert_lt\nfrom testifycompat import assert_lte\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tests import testingutils\nfrom tron import scheduler\nfrom tron.config import config_utils\nfrom tron.config import schedule_parse\nfrom tron.config.config_utils import NullConfigContext\nfrom tron.config.schedule_parse import parse_groc_expression\nfrom tron.utils import timeutils\n\n\nclass TestSchedulerFromConfig(TestCase):\n def test_cron_scheduler(self):\n line = \"cron */5 * * 7,8 *\"\n config_context = mock.Mock(path=\"test\")\n config = schedule_parse.valid_schedule(line, config_context)\n sched = scheduler.scheduler_from_config(config=config, time_zone=None)\n start_time = datetime.datetime(2012, 3, 14, 15, 9, 26)\n next_time = sched.next_run_time(start_time)\n assert_equal(next_time, datetime.datetime(2012, 7, 1, 0))\n assert_equal(str(sched), \"cron */5 * * 7,8 *\")\n\n def test_daily_scheduler(self):\n config_context = config_utils.NullConfigContext\n line = \"daily 17:32 MWF\"\n config = schedule_parse.valid_schedule(line, config_context)\n sched = scheduler.scheduler_from_config(config=config, time_zone=None)\n assert_equal(sched.time_spec.hours, [17])\n assert_equal(sched.time_spec.minutes, [32])\n start_time = datetime.datetime(2012, 3, 14, 15, 9, 26)\n for day in [14, 16, 19]:\n next_time = sched.next_run_time(start_time)\n assert_equal(next_time, datetime.datetime(2012, 3, day, 17, 32))\n start_time = next_time\n\n assert_equal(str(sched), \"daily 17:32 MWF\")\n\n\nclass GeneralSchedulerTestCase(testingutils.MockTimeTestCase):\n\n now = datetime.datetime.now().replace(hour=15, minute=0)\n\n def expected_time(self, date):\n return datetime.datetime.combine(date, datetime.time(14, 30))\n\n @setup\n def build_scheduler(self):\n self.scheduler = scheduler.GeneralScheduler(timestr=\"14:30\")\n one_day = datetime.timedelta(days=1)\n self.today = self.now.date()\n self.yesterday = self.now - one_day\n self.tomorrow = self.now + one_day\n\n def test_next_run_time(self):\n next_run = self.scheduler.next_run_time(timeutils.current_time())\n assert_equal(self.expected_time(self.tomorrow), next_run)\n\n next_run = self.scheduler.next_run_time(self.yesterday)\n assert_equal(self.expected_time(self.today), next_run)\n\n @mock.patch(\"tron.scheduler.get_jitter\", autospec=True)\n def test_next_run_time_with_jitter(self, mock_jitter):\n mock_jitter.return_value = delta = datetime.timedelta(seconds=-300)\n self.scheduler.jitter = datetime.timedelta(seconds=400)\n expected = self.expected_time(self.tomorrow) + delta\n next_run_time = self.scheduler.next_run_time(None)\n assert_equal(next_run_time, expected)\n\n def test__str__(self):\n assert_equal(str(self.scheduler), \"daily \")\n\n def test__str__with_jitter(self):\n self.scheduler.jitter = datetime.timedelta(seconds=300)\n assert_equal(str(self.scheduler), \"daily (+/- 0:05:00)\")\n\n\nclass GeneralSchedulerTimeTestBase(testingutils.MockTimeTestCase):\n\n now = datetime.datetime(2012, 3, 14, 15, 9, 26)\n\n @setup\n def build_scheduler(self):\n self.scheduler = scheduler.GeneralScheduler(timestr=\"14:30\")\n\n\nclass GeneralSchedulerTodayTest(GeneralSchedulerTimeTestBase):\n\n now = datetime.datetime.now().replace(hour=12, minute=0)\n\n def test(self):\n # If we schedule a job for later today, it should run today\n run_time = self.scheduler.next_run_time(self.now)\n next_run_date = run_time.date()\n\n assert_equal(next_run_date, self.now.date())\n earlier_time = datetime.datetime(\n self.now.year,\n self.now.month,\n self.now.day,\n hour=13,\n )\n assert_lte(earlier_time, run_time)\n\n\nclass GeneralSchedulerTomorrowTest(GeneralSchedulerTimeTestBase):\n\n now = datetime.datetime.now().replace(hour=15, minute=0)\n\n def test(self):\n # If we schedule a job for later today, it should run today\n run_time = self.scheduler.next_run_time(self.now)\n next_run_date = run_time.date()\n tomorrow = self.now.date() + datetime.timedelta(days=1)\n\n assert_equal(next_run_date, tomorrow)\n earlier_time = datetime.datetime(\n year=tomorrow.year,\n month=tomorrow.month,\n day=tomorrow.day,\n hour=13,\n )\n assert_lte(earlier_time, run_time)\n\n\nclass GeneralSchedulerLongJobRunTest(GeneralSchedulerTimeTestBase):\n\n now = datetime.datetime.now().replace(hour=12, minute=0)\n\n def test_long_jobs_dont_wedge_scheduler(self):\n # Advance days twice as fast as they are scheduled, demonstrating\n # that the scheduler will put things in the past if that's where\n # they belong, and run them as fast as possible\n\n last_run = self.scheduler.next_run_time(None)\n for i in range(10):\n next_run = self.scheduler.next_run_time(last_run)\n assert_equal(next_run, last_run + datetime.timedelta(days=1))\n\n self.now += datetime.timedelta(days=2)\n last_run = next_run\n\n\nclass GeneralSchedulerDSTTest(testingutils.MockTimeTestCase):\n\n now = datetime.datetime(2011, 11, 6, 1, 10, 0)\n now_utc = timeutils.current_time(tz=pytz.timezone(\"UTC\"))\n\n def hours_until_time(self, run_time, sch):\n tz = sch.time_zone\n now = timeutils.current_time()\n now = tz.localize(now) if tz else now\n seconds = timeutils.delta_total_seconds(run_time - now)\n return round(max(0, seconds) / 60 / 60, 1)\n\n def hours_diff_at_datetime(self, sch, *args, **kwargs):\n \"\"\"Return the number of hours until the next *two* runs of a job with\n the given scheduler\n \"\"\"\n self.now = datetime.datetime(*args, **kwargs)\n next_run = sch.next_run_time(self.now)\n t1 = self.hours_until_time(next_run, sch)\n next_run = sch.next_run_time(next_run.replace(tzinfo=None))\n t2 = self.hours_until_time(next_run, sch)\n return t1, t2\n\n def _assert_range(self, x, lower, upper):\n assert_gt(x, lower)\n assert_lt(x, upper)\n\n def test_fall_back(self):\n \"\"\"This test checks the behavior of the scheduler at the daylight\n savings time 'fall back' point, when the system time zone changes\n from (e.g.) PDT to PST.\n \"\"\"\n sch = scheduler.GeneralScheduler(time_zone=pytz.timezone(\"US/Pacific\"))\n\n # Exact crossover time:\n # datetime.datetime(2011, 11, 6, 9, 0, 0, tzinfo=pytz.utc)\n # This test will use times on either side of it.\n\n # From the PDT vantage point, the run time is 24.2 hours away:\n s1a, s1b = self.hours_diff_at_datetime(sch, 2011, 11, 6, 0, 50, 0)\n\n # From the PST vantage point, the run time is 22.8 hours away:\n # (this is measured from the point in absolute time 20 minutes after\n # the other measurement)\n s2a, s2b = self.hours_diff_at_datetime(sch, 2011, 11, 6, 1, 10, 0)\n\n self._assert_range(s1b - s1a, 23.99, 24.11)\n self._assert_range(s2b - s2a, 23.99, 24.11)\n self._assert_range(s1a - s2a, 1.39, 1.41)\n\n def test_correct_time(self):\n sch = scheduler.GeneralScheduler(time_zone=pytz.timezone(\"US/Pacific\"))\n next_run_time = sch.next_run_time(self.now)\n assert_equal(next_run_time.hour, 0)\n\n def test_spring_forward(self):\n \"\"\"This test checks the behavior of the scheduler at the daylight\n savings time 'spring forward' point, when the system time zone changes\n from (e.g.) PST to PDT.\n \"\"\"\n sch = scheduler.GeneralScheduler(time_zone=pytz.timezone(\"US/Pacific\"))\n\n # Exact crossover time:\n # datetime.datetime(2011, 3, 13, 2, 0, 0, tzinfo=pytz.utc)\n # This test will use times on either side of it.\n\n # From the PST vantage point, the run time is 20.2 hours away:\n s1a, s1b = self.hours_diff_at_datetime(sch, 2011, 3, 13, 2, 50, 0)\n\n # From the PDT vantage point, the run time is 20.8 hours away:\n # (this is measured from the point in absolute time 20 minutes after\n # the other measurement)\n s2a, s2b = self.hours_diff_at_datetime(sch, 2011, 3, 13, 3, 10, 0)\n\n self._assert_range(s1b - s1a, 23.99, 24.11)\n self._assert_range(s2b - s2a, 23.99, 24.11)\n self._assert_range(s1a - s2a, -0.61, -0.59)\n\n def test_handles_tz_specific_jobs_with_tz_specific_start_time(self):\n sch = scheduler.GeneralScheduler(time_zone=pytz.timezone(\"UTC\"))\n next_run_time = sch.next_run_time(self.now_utc)\n assert_equal(next_run_time.hour, 0)\n\n def test_handles_unsetting_the_time_zone(self):\n sch = scheduler.GeneralScheduler(time_zone=None)\n next_run_time = sch.next_run_time(self.now_utc)\n assert_equal(next_run_time.hour, 0)\n\n def test_handles_changing_the_time_zone(self):\n pacific_now = datetime.datetime.now(pytz.timezone(\"US/Pacific\"))\n pacific_offset = pacific_now.utcoffset().total_seconds() / 60 / 60\n sch = scheduler.GeneralScheduler(time_zone=pytz.timezone(\"US/Pacific\"))\n next_run_time = sch.next_run_time(self.now_utc)\n assert_equal(next_run_time.hour, -pacific_offset)\n\n\ndef parse_groc(config):\n config = schedule_parse.ConfigGenericSchedule(\"groc daily\", config, None)\n return parse_groc_expression(config, NullConfigContext)\n\n\ndef scheduler_from_config(config):\n return scheduler.scheduler_from_config(parse_groc(config), None)\n\n\nclass ComplexParserTest(testingutils.MockTimeTestCase):\n\n now = datetime.datetime(2011, 6, 1)\n\n def test_parse_all(self):\n config_string = \"1st,2nd,3rd,4th monday,Tue of march,apr,September at 00:00\"\n cfg = parse_groc(config_string)\n assert_equal(cfg.ordinals, {1, 2, 3, 4})\n assert_equal(cfg.monthdays, None)\n assert_equal(cfg.weekdays, {1, 2})\n assert_equal(cfg.months, {3, 4, 9})\n assert_equal(cfg.timestr, \"00:00\")\n assert_equal(\n scheduler_from_config(config_string),\n scheduler_from_config(config_string),\n )\n\n def test_parse_no_weekday(self):\n cfg = parse_groc(\n \"1st,2nd,3rd,10th day of march,apr,September at 00:00\",\n )\n assert_equal(cfg.ordinals, None)\n assert_equal(cfg.monthdays, {1, 2, 3, 10})\n assert_equal(cfg.weekdays, None)\n assert_equal(cfg.months, {3, 4, 9})\n assert_equal(cfg.timestr, \"00:00\")\n\n def test_parse_no_month(self):\n cfg = parse_groc(\"1st,2nd,3rd,10th day at 00:00\")\n assert_equal(cfg.ordinals, None)\n assert_equal(cfg.monthdays, {1, 2, 3, 10})\n assert_equal(cfg.weekdays, None)\n assert_equal(cfg.months, None)\n assert_equal(cfg.timestr, \"00:00\")\n\n def test_parse_monthly(self):\n for test_str in (\"1st day\", \"1st day of month\"):\n cfg = parse_groc(test_str)\n assert_equal(cfg.ordinals, None)\n assert_equal(cfg.monthdays, {1})\n assert_equal(cfg.weekdays, None)\n assert_equal(cfg.months, None)\n assert_equal(cfg.timestr, \"00:00\")\n\n def test_wildcards(self):\n cfg = parse_groc(\"every day\")\n assert_equal(cfg.ordinals, None)\n assert_equal(cfg.monthdays, None)\n assert_equal(cfg.weekdays, None)\n assert_equal(cfg.months, None)\n assert_equal(cfg.timestr, \"00:00\")\n\n def test_daily(self):\n sch = scheduler_from_config(\"every day\")\n next_run_date = sch.next_run_time(None)\n\n assert_gte(next_run_date, self.now)\n assert_equal(next_run_date.month, 6)\n assert_equal(next_run_date.day, 2)\n assert_equal(next_run_date.hour, 0)\n\n def test_daily_with_time(self):\n sch = scheduler_from_config(\"every day at 02:00\")\n next_run_date = sch.next_run_time(None)\n\n assert_gte(next_run_date, self.now)\n assert_equal(next_run_date.year, self.now.year)\n assert_equal(next_run_date.month, 6)\n assert_equal(next_run_date.day, 1)\n assert_equal(next_run_date.hour, 2)\n assert_equal(next_run_date.minute, 0)\n\n def test_weekly(self):\n sch = scheduler_from_config(\"every monday at 01:00\")\n next_run_date = sch.next_run_time(None)\n\n assert_gte(next_run_date, self.now)\n assert_equal(\n calendar.weekday(\n next_run_date.year,\n next_run_date.month,\n next_run_date.day,\n ),\n 0,\n )\n\n def test_weekly_in_month(self):\n sch = scheduler_from_config(\"every monday of January at 00:01\")\n next_run_date = sch.next_run_time(None)\n\n assert_gte(next_run_date, self.now)\n assert_equal(next_run_date.year, self.now.year + 1)\n assert_equal(next_run_date.month, 1)\n assert_equal(next_run_date.hour, 0)\n assert_equal(next_run_date.minute, 1)\n assert_equal(\n calendar.weekday(\n next_run_date.year,\n next_run_date.month,\n next_run_date.day,\n ),\n 0,\n )\n\n def test_monthly(self):\n sch = scheduler_from_config(\"1st day\")\n next_run_date = sch.next_run_time(None)\n\n assert_gt(next_run_date, self.now)\n assert_equal(next_run_date.month, 7)\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/serialize/__init__.py", "content": "" }, { "path": "tests/serialize/filehandler_test.py", "content": "import os\nimport shutil\nimport time\nfrom tempfile import mkdtemp\nfrom tempfile import NamedTemporaryFile\nfrom unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_in\nfrom testifycompat import assert_not_equal\nfrom testifycompat import assert_not_in\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import suite\nfrom testifycompat import teardown\nfrom testifycompat import TestCase\nfrom tron.serialize.filehandler import FileHandleManager\nfrom tron.serialize.filehandler import NullFileHandle\nfrom tron.serialize.filehandler import OutputPath\nfrom tron.serialize.filehandler import OutputStreamSerializer\n\n\nclass TestFileHandleWrapper(TestCase):\n @setup\n def setup_fh_wrapper(self):\n self.file = NamedTemporaryFile(\"r\")\n self.manager = FileHandleManager.get_instance()\n self.fh_wrapper = self.manager.open(self.file.name)\n\n @teardown\n def teardown_fh_wrapper(self):\n self.fh_wrapper.close()\n FileHandleManager.reset()\n\n def test_init(self):\n assert_equal(self.fh_wrapper._fh, NullFileHandle)\n\n def test_close(self):\n # Test close without a write, no exception is good\n self.fh_wrapper.close()\n # Test close again, after already closed\n self.fh_wrapper.close()\n\n def test_close_with_write(self):\n # Test close with a write\n self.fh_wrapper.write(\"some things\")\n self.fh_wrapper.close()\n assert_equal(self.fh_wrapper._fh, NullFileHandle)\n assert_equal(self.fh_wrapper.manager, self.manager)\n # This is somewhat coupled\n assert_not_in(self.fh_wrapper, self.manager.cache)\n\n def test_write(self):\n # Test write without a previous open\n before_time = time.time()\n self.fh_wrapper.write(\"some things\")\n after_time = time.time()\n\n assert self.fh_wrapper._fh\n assert_equal(self.fh_wrapper._fh.closed, False)\n assert before_time <= self.fh_wrapper.last_accessed <= after_time\n\n # Test write after previous open\n before_time = time.time()\n self.fh_wrapper.write(\"\\nmore things\")\n after_time = time.time()\n assert before_time <= self.fh_wrapper.last_accessed <= after_time\n self.fh_wrapper.close()\n with open(self.file.name) as fh:\n assert_equal(fh.read(), \"some things\\nmore things\")\n\n def test_close_many(self):\n self.fh_wrapper.write(\"some things\")\n self.fh_wrapper.close()\n self.fh_wrapper.close()\n\n def test_context_manager(self):\n with self.fh_wrapper as fh:\n fh.write(\"123\")\n assert fh._fh is None\n with open(self.file.name) as fh:\n assert_equal(fh.read(), \"123\")\n\n\nclass TestFileHandleManager(TestCase):\n @setup\n def setup_fh_manager(self):\n FileHandleManager.reset()\n self.file1 = NamedTemporaryFile(\"r\")\n self.file2 = NamedTemporaryFile(\"r\")\n FileHandleManager.set_max_idle_time(2)\n self.manager = FileHandleManager.get_instance()\n\n @teardown\n def teardown_fh_manager(self):\n FileHandleManager.reset()\n\n def test_get_instance(self):\n assert_equal(self.manager, FileHandleManager.get_instance())\n # Repeat for good measure\n assert_equal(self.manager, FileHandleManager.get_instance())\n\n def test_set_max_idle_time(self):\n max_idle_time = 300\n FileHandleManager.set_max_idle_time(max_idle_time)\n assert_equal(max_idle_time, self.manager.max_idle_time)\n\n def test_open(self):\n # Not yet in cache\n fh_wrapper = self.manager.open(self.file1.name)\n assert_in(fh_wrapper.name, self.manager.cache)\n\n # Should now be in cache\n fh_wrapper2 = self.manager.open(self.file1.name)\n\n # Same wrapper\n assert_equal(fh_wrapper, fh_wrapper2)\n\n # Different wrapper\n assert_not_equal(fh_wrapper, self.manager.open(self.file2.name))\n\n def test_cleanup_none(self):\n # Nothing to remove\n fh_wrapper = self.manager.open(self.file1.name)\n self.manager.cleanup()\n assert_in(fh_wrapper.name, self.manager.cache)\n\n def test_cleanup_single(self):\n fh_wrapper = self.manager.open(self.file1.name)\n fh_wrapper.last_accessed = 123456\n\n def time_func():\n return 123458.1\n\n self.manager.cleanup(time_func)\n assert_not_in(fh_wrapper.name, self.manager.cache)\n assert_equal(len(self.manager.cache), 0)\n\n def test_cleanup_many(self):\n fh_wrappers = [\n self.manager.open(self.file1.name),\n self.manager.open(self.file2.name),\n self.manager.open(NamedTemporaryFile(\"r\").name),\n self.manager.open(NamedTemporaryFile(\"r\").name),\n self.manager.open(NamedTemporaryFile(\"r\").name),\n ]\n for i, fh_wrapper in enumerate(fh_wrappers):\n fh_wrapper.last_accessed = 123456 + i\n\n def time_func():\n return 123460.1\n\n self.manager.cleanup(time_func)\n assert_equal(len(self.manager.cache), 2)\n\n for fh_wrapper in fh_wrappers[:3]:\n assert_not_in(fh_wrapper.name, self.manager.cache)\n\n for fh_wrapper in fh_wrappers[3:]:\n assert_in(fh_wrapper.name, self.manager.cache)\n\n def test_cleanup_opened(self):\n fh_wrapper = self.manager.open(self.file1.name)\n fh_wrapper.write(\"Some things\")\n\n fh_wrapper.last_accessed = 123456\n\n def time_func():\n return 123458.1\n\n self.manager.cleanup(time_func)\n assert_not_in(fh_wrapper.name, self.manager.cache)\n assert_equal(len(self.manager.cache), 0)\n\n def test_cleanup_natural(self):\n FileHandleManager.set_max_idle_time(1)\n fh_wrapper1 = self.manager.open(self.file1.name)\n fh_wrapper2 = self.manager.open(self.file2.name)\n fh_wrapper1.write(\"Some things\")\n\n time.sleep(1.5)\n fh_wrapper2.write(\"Other things.\")\n\n assert_not_in(fh_wrapper1.name, self.manager.cache)\n assert_in(fh_wrapper2.name, self.manager.cache)\n\n # Now that 1 is closed, try writing again\n fh_wrapper1.write(\"Some things\")\n assert_in(fh_wrapper1.name, self.manager.cache)\n assert not fh_wrapper1._fh.closed\n\n def test_remove(self):\n # In cache\n fh_wrapper = self.manager.open(self.file1.name)\n assert_in(fh_wrapper.name, self.manager.cache)\n self.manager.remove(fh_wrapper)\n assert_not_in(fh_wrapper.name, self.manager.cache)\n\n # Not in cache\n self.manager.remove(fh_wrapper)\n assert_not_in(fh_wrapper.name, self.manager.cache)\n\n def test_update(self):\n fh_wrapper1 = self.manager.open(self.file1.name)\n fh_wrapper2 = self.manager.open(self.file2.name)\n assert_equal(\n list(self.manager.cache.keys()),\n [fh_wrapper1.name, fh_wrapper2.name],\n )\n\n self.manager.update(fh_wrapper1)\n assert_equal(\n list(self.manager.cache.keys()),\n [fh_wrapper2.name, fh_wrapper1.name],\n )\n\n\nclass TestOutputStreamSerializer(TestCase):\n @setup\n def setup_serializer(self):\n self.test_dir = mkdtemp()\n self.serial = OutputStreamSerializer([self.test_dir])\n self.filename = \"STARS\"\n self.content = \"123\\n456\\n789\"\n self.expected = [line for line in self.content.split(\"\\n\")]\n\n @teardown\n def teardown_test_dir(self):\n shutil.rmtree(self.test_dir)\n\n def _write_contents(self):\n with open(self.serial.full_path(self.filename), \"w\") as f:\n f.write(self.content)\n\n def test_open(self):\n with self.serial.open(self.filename) as fh:\n fh.write(self.content)\n\n with open(self.serial.full_path(self.filename)) as f:\n assert_equal(f.read(), self.content)\n\n @suite(\"integration\")\n def test_init_with_output_path(self):\n path = OutputPath(self.test_dir, \"one\", \"two\", \"three\")\n stream = OutputStreamSerializer(path)\n assert_equal(stream.base_path, str(path))\n\n def test_tail(self):\n self._write_contents()\n assert_equal(self.serial.tail(self.filename), self.expected)\n\n def test_tail_num_lines(self):\n self._write_contents()\n assert_equal(self.serial.tail(self.filename, 1), self.expected[-1:])\n\n def test_tail_file_does_not_exist(self):\n file_dne = \"bogusfile123\"\n assert_equal(self.serial.tail(file_dne), [])\n\n\nclass TestOutputPath(TestCase):\n @setup\n def setup_path(self):\n self.path = OutputPath(\"one\", \"two\", \"three\")\n\n def test__init__(self):\n assert_equal(self.path.base, \"one\")\n assert_equal(self.path.parts, [\"two\", \"three\"])\n\n path = OutputPath(\"base\")\n assert_equal(path.base, \"base\")\n assert_equal(path.parts, [])\n\n def test__iter__(self):\n assert_equal(list(self.path), [\"one\", \"two\", \"three\"])\n\n def test__str__(self):\n # Breaks in windows probably,\n assert_equal(\"one/two/three\", str(self.path))\n\n def test_append(self):\n self.path.append(\"four\")\n assert_equal(self.path.parts, [\"two\", \"three\", \"four\"])\n\n def test_clone(self):\n new_path = self.path.clone()\n assert_equal(str(new_path), str(self.path))\n\n self.path.append(\"alpha\")\n assert_equal(str(new_path), \"one/two/three\")\n\n new_path.append(\"beta\")\n assert_equal(str(self.path), \"one/two/three/alpha\")\n\n def test_clone_with_parts(self):\n new_path = self.path.clone(\"seven\", \"ten\")\n assert_equal(list(new_path), [\"one/two/three\", \"seven\", \"ten\"])\n\n def test_delete(self):\n tmp_dir = mkdtemp()\n path = OutputPath(tmp_dir)\n path.delete()\n assert not os.path.exists(tmp_dir)\n\n def test__eq__(self):\n other = mock.MagicMock(base=\"one\", parts=[\"two\", \"three\"])\n assert_equal(self.path, other)\n\n def test__ne__(self):\n other = mock.MagicMock(base=\"one/two\", parts=[\"three\"])\n assert_not_equal(self.path, other)\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/serialize/runstate/__init__.py", "content": "" }, { "path": "tests/serialize/runstate/dynamodb_state_store_test.py", "content": "import gzip\nimport json\nfrom unittest import mock\n\nimport boto3\nimport pytest\nfrom boto3.dynamodb.types import Binary\nfrom moto import mock_dynamodb\nfrom moto.dynamodb.responses import dynamo_json_dump\n\nfrom tron.serialize.runstate.dynamodb_state_store import DynamoDBStateStore\nfrom tron.serialize.runstate.dynamodb_state_store import MAX_UNPROCESSED_KEYS_RETRIES\n\n\ndef mock_transact_write_items(self):\n \"\"\"\n This mocks moto.dynamodb2.responses.DynamoHandler.transact_write_items,\n which is used to mock dynamodb client. This function calls put_item,\n update_item, and delete_item based on the arguments of transact_write_item.\n \"\"\"\n\n def put_item(item):\n name = item[\"TableName\"]\n record = item[\"Item\"]\n return self.dynamodb_backend.put_item(name, record)\n\n def delete_item(item):\n name = item[\"TableName\"]\n keys = item[\"Key\"]\n return self.dynamodb_backend.delete_item(name, keys)\n\n def update_item(item):\n name = item[\"TableName\"]\n key = item[\"Key\"]\n update_expression = item.get(\"UpdateExpression\")\n attribute_updates = item.get(\"AttributeUpdates\")\n expression_attribute_names = item.get(\"ExpressionAttributeNames\", {})\n expression_attribute_values = item.get(\"ExpressionAttributeValues\", {})\n return self.dynamodb_backend.update_item(\n name,\n key,\n update_expression,\n attribute_updates,\n expression_attribute_names,\n expression_attribute_values,\n )\n\n transact_items = self.body[\"TransactItems\"]\n\n for transact_item in transact_items:\n if \"Put\" in transact_item:\n put_item(transact_item[\"Put\"])\n elif \"Update\" in transact_item:\n update_item(transact_item[\"Update\"])\n elif \"Delete\" in transact_item:\n delete_item(transact_item[\"Delete\"])\n\n return dynamo_json_dump({})\n\n\n@pytest.fixture(autouse=True)\ndef store():\n with mock.patch(\n \"moto.dynamodb.responses.DynamoHandler.transact_write_items\",\n new=mock_transact_write_items,\n create=True,\n ), mock_dynamodb():\n dynamodb = boto3.resource(\"dynamodb\", region_name=\"us-west-2\")\n table_name = \"tmp\"\n store = DynamoDBStateStore(table_name, \"us-west-2\", stopping=True)\n store.table = dynamodb.create_table(\n TableName=table_name,\n KeySchema=[\n {\n \"AttributeName\": \"key\",\n \"KeyType\": \"HASH\",\n }, # Partition key\n {\n \"AttributeName\": \"index\",\n \"KeyType\": \"RANGE\",\n }, # Sort key\n ],\n AttributeDefinitions=[\n {\n \"AttributeName\": \"key\",\n \"AttributeType\": \"S\",\n },\n {\n \"AttributeName\": \"index\",\n \"AttributeType\": \"N\",\n },\n ],\n ProvisionedThroughput={\n \"ReadCapacityUnits\": 10,\n \"WriteCapacityUnits\": 10,\n },\n )\n store.client = boto3.client(\"dynamodb\", region_name=\"us-west-2\")\n # Has to be yield here for moto to work\n yield store\n\n\n@pytest.fixture\ndef small_job():\n yield {\n \"enabled\": True,\n \"run_nums\": [1],\n }\n\n\n@pytest.fixture\ndef small_object():\n yield {\n \"job_name\": \"example_job\",\n \"run_num\": 1,\n \"run_time\": None,\n \"time_zone\": None,\n \"node_name\": \"example_node\",\n \"runs\": [],\n \"cleanup_run\": None,\n \"manual\": False,\n }\n\n\n@pytest.fixture\ndef large_object():\n # We need this object to exceed OBJECT_SIZE after gzip compression so it\n # requires multiple partitions. We pad node_name because JobRun.to_json()\n # only serializes known fields — arbitrary fields like \"large_data\" would\n # be silently dropped. Ideally we'd use a more realistic field like\n # \"runs\", but each action run requires ~15 nested fields, making the\n # fixture unwieldy. node_name is the simplest string field that survives\n # the JSON round-trip.\n yield {\n \"job_name\": \"example_job\",\n \"run_num\": 1,\n \"run_time\": None,\n \"time_zone\": None,\n \"node_name\": \"\".join(str(i) for i in range(200_000)),\n \"runs\": [],\n \"cleanup_run\": None,\n \"manual\": False,\n }\n\n\n@pytest.mark.usefixtures(\"store\")\nclass TestDynamoDBStateStore:\n def test_save(self, store, small_job, small_object):\n key_value_pairs = [\n (\n store.build_key(\"job_state\", \"two\"),\n small_job,\n ),\n (\n store.build_key(\"job_run_state\", \"four\"),\n small_object,\n ),\n ]\n store.save(key_value_pairs)\n store._consume_save_queue()\n\n assert store.save_errors == 0\n keys = [\n store.build_key(\"job_state\", \"two\"),\n store.build_key(\"job_run_state\", \"four\"),\n ]\n with mock.patch(\"tron.config.static_config.load_yaml_file\", autospec=True), mock.patch(\n \"tron.config.static_config.build_configuration_watcher\", autospec=True\n ):\n vals = store.restore(keys)\n for key, value in key_value_pairs:\n assert vals[key] == value\n\n expected_values = {keys[0]: small_job, keys[1]: small_object}\n for key in keys:\n item = store.table.get_item(Key={\"key\": key, \"index\": 0})\n assert \"Item\" in item\n assert \"json_val\" in item[\"Item\"]\n\n compressed_val = item[\"Item\"][\"json_val\"]\n assert isinstance(compressed_val, Binary)\n\n decompressed_json = gzip.decompress(compressed_val.value)\n assert json.loads(decompressed_json) == expected_values[key]\n\n def test_save_multi_partition_object(self, store, large_object):\n key_value_pairs = [\n (\n store.build_key(\"job_run_state\", \"two\"),\n large_object,\n ),\n ]\n store.save(key_value_pairs)\n store._consume_save_queue()\n\n assert store.save_errors == 0\n keys = [store.build_key(\"job_run_state\", \"two\")]\n\n with mock.patch(\"tron.config.static_config.load_yaml_file\", autospec=True), mock.patch(\n \"tron.config.static_config.build_configuration_watcher\", autospec=True\n ):\n vals = store.restore(keys)\n for key, value in key_value_pairs:\n assert vals[key] == value\n\n def test_restore(self, store, small_object):\n keys = [store.build_key(\"job_run_state\", i) for i in range(3)]\n value = small_object\n pairs = zip(keys, (value for i in range(len(keys))))\n store.save(pairs)\n store._consume_save_queue()\n\n assert store.save_errors == 0\n with mock.patch(\"tron.config.static_config.load_yaml_file\", autospec=True), mock.patch(\n \"tron.config.static_config.build_configuration_watcher\", autospec=True\n ):\n vals = store.restore(keys)\n for key in keys:\n assert vals[key] == small_object\n\n def test_restore_multi_partition_object(self, store, large_object):\n keys = [store.build_key(\"job_run_state\", i) for i in range(3)]\n value = large_object\n pairs = zip(keys, (value for i in range(len(keys))))\n store.save(pairs)\n store._consume_save_queue()\n\n assert store.save_errors == 0\n\n for key in keys:\n num_partitions, num_json_val_partitions = store._get_num_of_partitions(key)\n assert num_json_val_partitions > 1\n assert num_partitions > 1\n\n with mock.patch(\"tron.config.static_config.load_yaml_file\", autospec=True), mock.patch(\n \"tron.config.static_config.build_configuration_watcher\", autospec=True\n ):\n vals = store.restore(keys)\n for key in keys:\n assert vals[key] == large_object\n\n def test_delete_item(self, store, small_object):\n keys = [store.build_key(\"job_state\", i) for i in range(3)]\n pairs = list(zip(keys, (small_object for _ in range(len(keys)))))\n\n store.save(pairs)\n store._consume_save_queue()\n\n for key, _ in pairs:\n store._delete_item(key)\n\n for key, _ in pairs:\n num_partitions, num_json_val_partitions = store._get_num_of_partitions(key)\n assert num_partitions == 0\n assert num_json_val_partitions == 0\n\n def test_delete_multi_partition_item(self, store, large_object):\n keys = [store.build_key(\"job_state\", i) for i in range(3)]\n pairs = list(zip(keys, (large_object for _ in range(len(keys)))))\n\n store.save(pairs)\n store._consume_save_queue()\n\n for key, _ in pairs:\n num_partitions, num_json_val_partitions = store._get_num_of_partitions(key)\n assert num_partitions > 1\n assert num_json_val_partitions > 1\n\n for key, _ in pairs:\n store._delete_item(key)\n\n for key, _ in pairs:\n num_partitions, num_json_val_partitions = store._get_num_of_partitions(key)\n assert num_partitions == 0\n assert num_json_val_partitions == 0\n\n def test_delete_if_val_is_none(self, store, small_object):\n key_value_pairs = [\n (\n store.build_key(\"job_state\", \"two\"),\n small_object,\n ),\n (\n store.build_key(\"job_run_state\", \"four\"),\n small_object,\n ),\n ]\n store.save(key_value_pairs)\n store._consume_save_queue()\n\n delete = [\n (\n store.build_key(\"job_state\", \"two\"),\n None,\n ),\n ]\n store.save(delete)\n store._consume_save_queue()\n\n assert store.save_errors == 0\n # Try to restore both, we should just get one back\n keys = [\n store.build_key(\"job_state\", \"two\"),\n store.build_key(\"job_run_state\", \"four\"),\n ]\n with mock.patch(\"tron.config.static_config.load_yaml_file\", autospec=True), mock.patch(\n \"tron.config.static_config.build_configuration_watcher\", autospec=True\n ):\n vals = store.restore(keys)\n assert vals == {\"job_run_state four\": small_object}\n\n @pytest.mark.parametrize(\n \"test_object, side_effects, expected_save_errors, expected_queue_length\",\n [\n # All attempts fail\n (\"small_object\", [KeyError(\"foo\")] * 3, 3, 1),\n (\"large_object\", [KeyError(\"foo\")] * 3, 3, 1),\n # Failure followed by success\n (\"small_object\", [KeyError(\"foo\"), {}], 0, 0),\n (\"large_object\", [KeyError(\"foo\")] + [{}] * 10, 0, 0),\n ],\n )\n def test_retry_saving(\n self, test_object, side_effects, expected_save_errors, expected_queue_length, store, small_object, large_object\n ):\n object_mapping = {\n \"small_object\": small_object,\n \"large_object\": large_object,\n }\n value = object_mapping[test_object]\n\n with mock.patch.object(\n store.client,\n \"transact_write_items\",\n side_effect=side_effects,\n ) as mock_transact_write:\n keys = [store.build_key(\"job_state\", 0)]\n pairs = zip(keys, [value])\n store.save(pairs)\n\n for _ in side_effects:\n store._consume_save_queue()\n\n assert mock_transact_write.called\n assert store.save_errors == expected_save_errors\n assert len(store.save_queue) == expected_queue_length\n\n @pytest.mark.parametrize(\n \"attempt, expected_delay\",\n [\n (1, 1),\n (2, 2),\n (3, 4),\n (4, 8),\n (5, 10),\n (6, 10),\n (7, 10),\n ],\n )\n def test_calculate_backoff_delay(self, store, attempt, expected_delay):\n delay = store._calculate_backoff_delay(attempt)\n assert delay == expected_delay\n\n def test_retry_reading(self, store):\n unprocessed_value = {\n \"Responses\": {},\n \"UnprocessedKeys\": {\n store.name: {\n \"Keys\": [{\"key\": {\"S\": store.build_key(\"job_state\", 0)}, \"index\": {\"N\": \"0\"}}],\n \"ConsistentRead\": True,\n }\n },\n }\n\n keys = [store.build_key(\"job_state\", 0)]\n\n with mock.patch.object(\n store.client,\n \"batch_get_item\",\n return_value=unprocessed_value,\n ) as mock_batch_get_item, mock.patch(\"time.sleep\") as mock_sleep, pytest.raises(Exception) as exec_info:\n store.restore(keys)\n assert \"failed to retrieve items with keys\" in str(exec_info.value)\n assert mock_batch_get_item.call_count == MAX_UNPROCESSED_KEYS_RETRIES\n assert mock_sleep.call_count == MAX_UNPROCESSED_KEYS_RETRIES\n\n def test_restore_exception_propagation(self, store):\n # This test is to ensure that restore propagates exceptions upwards: see DAR-2328\n keys = [store.build_key(\"job_state\", i) for i in range(3)]\n\n mock_future = mock.MagicMock()\n mock_future.result.side_effect = Exception(\"mocked exception\")\n with mock.patch(\"concurrent.futures.Future\", return_value=mock_future, autospec=True):\n with mock.patch(\"concurrent.futures.as_completed\", return_value=[mock_future], autospec=True):\n with pytest.raises(Exception) as exec_info, mock.patch(\n \"tron.config.static_config.load_yaml_file\", autospec=True\n ), mock.patch(\"tron.config.static_config.build_configuration_watcher\", autospec=True):\n store.restore(keys)\n assert str(exec_info.value) == \"mocked exception\"\n\n def test_serialization_failure_preserves_existing_row(self, store, small_object):\n \"\"\"When json serialization fails, the existing DynamoDB row should not be deleted.\"\"\"\n key = store.build_key(\"job_run_state\", \"preserve_me\")\n\n store.save([(key, small_object)])\n store._consume_save_queue()\n assert store.save_errors == 0\n\n new_val = {**small_object, \"run_num\": 999}\n with mock.patch.object(store, \"_serialize_item\", return_value=None):\n store.save([(key, new_val)])\n\n # The save() call above already called _serialize_item (which returned None),\n # so the queue has (new_val, None). _consume_save_queue should retry and still fail.\n with mock.patch.object(store, \"_serialize_item\", return_value=None):\n store._consume_save_queue()\n\n # The item should be requeued\n assert len(store.save_queue) == 1\n assert store.save_errors == 1\n\n # The original key should still be intact\n with mock.patch(\"tron.config.static_config.load_yaml_file\", autospec=True), mock.patch(\n \"tron.config.static_config.build_configuration_watcher\", autospec=True\n ):\n vals = store.restore([key])\n assert vals[key] == small_object\n\n def test_serialization_retry_succeeds(self, store, small_object):\n \"\"\"When initial json_val is None but retry succeeds, the item should be saved normally.\"\"\"\n key = store.build_key(\"job_run_state\", \"retry_ok\")\n\n with mock.patch.object(store, \"_serialize_item\", return_value=None):\n store.save([(key, small_object)])\n\n queued_val, queued_json = store.save_queue[key]\n assert queued_val == small_object\n assert queued_json is None\n\n # This time it should succeed\n store._consume_save_queue()\n\n assert len(store.save_queue) == 0\n assert store.save_errors == 0\n\n with mock.patch(\"tron.config.static_config.load_yaml_file\", autospec=True), mock.patch(\n \"tron.config.static_config.build_configuration_watcher\", autospec=True\n ):\n vals = store.restore([key])\n assert vals[key] == small_object\n\n def test_delete_sentinel_proceeds_with_deletion(self, store, small_object):\n \"\"\"When val is None the row should be deleted.\"\"\"\n key = store.build_key(\"job_run_state\", \"delete_me\")\n\n store.save([(key, small_object)])\n store._consume_save_queue()\n assert store.save_errors == 0\n\n store.save([(key, None)])\n store._consume_save_queue()\n assert store.save_errors == 0\n\n num_partitions, num_json_val_partitions = store._get_num_of_partitions(key)\n assert num_partitions == 0\n assert num_json_val_partitions == 0\n" }, { "path": "tests/serialize/runstate/shelvestore_test.py", "content": "import os\nimport shutil\nimport tempfile\n\nfrom testifycompat import assert_equal\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import teardown\nfrom testifycompat import TestCase\nfrom tron.serialize.runstate.shelvestore import Py2Shelf\nfrom tron.serialize.runstate.shelvestore import ShelveKey\nfrom tron.serialize.runstate.shelvestore import ShelveStateStore\n\n\nclass TestShelveStateStore(TestCase):\n @setup\n def setup_store(self):\n self.tmpdir = tempfile.mkdtemp()\n self.filename = os.path.join(self.tmpdir, \"state\")\n self.store = ShelveStateStore(self.filename)\n\n @teardown\n def teardown_store(self):\n shutil.rmtree(self.tmpdir)\n\n def test__init__(self):\n assert_equal(self.filename, self.store.filename)\n\n def test_save(self):\n key_value_pairs = [\n (\n ShelveKey(\"one\", \"two\"),\n {\n \"this\": \"data\",\n },\n ),\n (\n ShelveKey(\"three\", \"four\"),\n {\n \"this\": \"data2\",\n },\n ),\n ]\n self.store.save(key_value_pairs)\n self.store.cleanup()\n\n stored_data = Py2Shelf(self.filename)\n for key, value in key_value_pairs:\n assert_equal(stored_data[str(key.key)], value)\n stored_data.close()\n\n def test_delete(self):\n key_value_pairs = [\n (\n ShelveKey(\"one\", \"two\"),\n {\n \"this\": \"data\",\n },\n ),\n (\n ShelveKey(\"three\", \"four\"),\n {\n \"this\": \"data2\",\n },\n ),\n # Delete first key\n (\n ShelveKey(\"one\", \"two\"),\n None,\n ),\n ]\n self.store.save(key_value_pairs)\n self.store.cleanup()\n\n stored_data = Py2Shelf(self.filename)\n assert stored_data == {\n str(ShelveKey(\"three\", \"four\").key): {\"this\": \"data2\"},\n }\n stored_data.close()\n\n def test_restore(self):\n self.store.cleanup()\n keys = [ShelveKey(\"thing\", i) for i in range(5)]\n value = {\"this\": \"data\"}\n store = Py2Shelf(self.filename)\n for key in keys:\n store[str(key.key)] = value\n store.close()\n\n self.store.shelve = Py2Shelf(self.filename)\n retrieved_data = self.store.restore(keys)\n for key in keys:\n assert_equal(retrieved_data[key], value)\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/serialize/runstate/statemanager_test.py", "content": "import os\nimport shutil\nimport tempfile\nfrom unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tests.assertions import assert_raises\nfrom tests.testingutils import autospec_method\nfrom tron.config import schema\nfrom tron.core.job import Job\nfrom tron.core.jobrun import JobRun\nfrom tron.mesos import MesosClusterRepository\nfrom tron.serialize import runstate\nfrom tron.serialize.runstate.shelvestore import ShelveStateStore\nfrom tron.serialize.runstate.statemanager import PersistenceManagerFactory\nfrom tron.serialize.runstate.statemanager import PersistenceStoreError\nfrom tron.serialize.runstate.statemanager import PersistentStateManager\nfrom tron.serialize.runstate.statemanager import StateChangeWatcher\nfrom tron.serialize.runstate.statemanager import StateSaveBuffer\n\n\nclass TestPersistenceManagerFactory(TestCase):\n def test_from_config_shelve(self):\n tmpdir = tempfile.mkdtemp()\n try:\n fname = os.path.join(tmpdir, \"state\")\n config = schema.ConfigState(\n store_type=\"shelve\",\n name=fname,\n buffer_size=0,\n )\n manager = PersistenceManagerFactory.from_config(config)\n store = manager._impl\n assert_equal(store.filename, config.name)\n assert isinstance(store, ShelveStateStore)\n finally:\n shutil.rmtree(tmpdir)\n\n\nclass TestStateSaveBuffer(TestCase):\n @setup\n def setup_buffer(self):\n self.buffer_size = 5\n self.buffer = StateSaveBuffer(self.buffer_size)\n\n def test_save(self):\n assert self.buffer.save(1, 2)\n assert not self.buffer.save(1, 3)\n assert not self.buffer.save(1, 4)\n assert not self.buffer.save(1, 5)\n assert not self.buffer.save(1, 6)\n assert self.buffer.save(1, 7)\n assert_equal(self.buffer.buffer[1], 7)\n\n def test__iter__(self):\n self.buffer.save(1, 2)\n self.buffer.save(2, 3)\n items = list(self.buffer)\n assert not self.buffer.buffer\n assert_equal(items, [(1, 2), (2, 3)])\n\n\nclass TestPersistentStateManager(TestCase):\n @setup\n def setup_manager(self):\n self.store = mock.Mock()\n self.store.build_key.side_effect = lambda t, i: f\"{t}{i}\"\n self.buffer = StateSaveBuffer(1)\n self.manager = PersistentStateManager(self.store, self.buffer)\n\n def test__init__(self):\n assert_equal(self.manager._impl, self.store)\n\n def test_keys_for_items(self):\n names = [\"namea\", \"nameb\"]\n key_to_item_map = self.manager._keys_for_items(\"type\", names)\n\n keys = [\"type%s\" % name for name in names]\n assert_equal(key_to_item_map, dict(zip(keys, names)))\n\n def test_restore(self):\n job_names = [\"one\", \"two\"]\n with mock.patch.object(self.manager, \"_restore_dicts\", autospec=True,) as mock_restore_dicts, mock.patch.object(\n self.manager,\n \"_restore_runs_for_job\",\n autospec=True,\n ) as mock_restore_runs:\n mock_restore_dicts.side_effect = [\n # _restore_dicts for JOB_STATE\n {\n \"one\": {\"key\": \"val1\"},\n \"two\": {\"key\": \"val2\"},\n },\n ]\n\n restored_state = self.manager.restore(job_names)\n assert mock_restore_dicts.call_args_list == [\n mock.call(runstate.JOB_STATE, job_names),\n ]\n assert len(mock_restore_runs.call_args_list) == 2\n assert restored_state == {\n runstate.JOB_STATE: {\n \"one\": {\"key\": \"val1\", \"runs\": mock_restore_runs.return_value},\n \"two\": {\"key\": \"val2\", \"runs\": mock_restore_runs.return_value},\n },\n }\n\n def test_restore_runs_for_job(self):\n job_state = {\"run_nums\": [2, 3], \"enabled\": True}\n with mock.patch.object(\n self.manager,\n \"_restore_dicts\",\n autospec=True,\n ) as mock_restore_dicts:\n mock_restore_dicts.side_effect = [\n {\"job_a.2\": {\"job_name\": \"job_a\", \"run_num\": 2}, \"job_a.3\": {\"job_name\": \"job_a\", \"run_num\": 3}}\n ]\n runs = self.manager._restore_runs_for_job(\"job_a\", job_state)\n\n assert mock_restore_dicts.call_args_list == [mock.call(runstate.JOB_RUN_STATE, [\"job_a.2\", \"job_a.3\"])]\n assert runs == [{\"job_name\": \"job_a\", \"run_num\": 3}, {\"job_name\": \"job_a\", \"run_num\": 2}]\n\n def test_restore_runs_for_job_one_missing(self):\n job_state = {\"run_nums\": [2, 3], \"enabled\": True}\n with mock.patch.object(\n self.manager,\n \"_restore_dicts\",\n autospec=True,\n ) as mock_restore_dicts:\n mock_restore_dicts.side_effect = [{\"job_a.3\": {\"job_name\": \"job_a\", \"run_num\": 3}, \"job_b\": {}}]\n runs = self.manager._restore_runs_for_job(\"job_a\", job_state)\n\n assert mock_restore_dicts.call_args_list == [\n mock.call(runstate.JOB_RUN_STATE, [\"job_a.2\", \"job_a.3\"]),\n ]\n assert runs == [{\"job_name\": \"job_a\", \"run_num\": 3}]\n\n def test_restore_dicts(self):\n names = [\"namea\", \"nameb\"]\n autospec_method(self.manager._keys_for_items)\n self.manager._keys_for_items.return_value = dict(enumerate(names))\n self.store.restore.return_value = {\n 0: {\n \"state\": \"data\",\n },\n 1: {\n \"state\": \"2data\",\n },\n }\n state_data = self.manager._restore_dicts(\"type\", names)\n expected = {\n names[0]: {\n \"state\": \"data\",\n },\n names[1]: {\n \"state\": \"2data\",\n },\n }\n assert_equal(expected, state_data)\n\n def test_save(self):\n name, state_data = \"name\", mock.Mock()\n self.manager.save(runstate.JOB_STATE, name, state_data)\n key = f\"{runstate.JOB_STATE}{name}\"\n self.store.save.assert_called_with([(key, state_data)])\n\n def test_save_failed(self):\n self.store.save.side_effect = PersistenceStoreError(\"blah\")\n assert_raises(\n PersistenceStoreError,\n self.manager.save,\n None,\n None,\n None,\n )\n\n def test_save_while_disabled(self):\n with self.manager.disabled():\n self.manager.save(\"something\", \"name\", mock.Mock())\n assert not self.store.save.mock_calls\n\n def test_delete(self):\n name = \"name\"\n self.manager.delete(runstate.JOB_STATE, name)\n key = f\"{runstate.JOB_STATE}{name}\"\n self.store.save.assert_called_with([(key, None)])\n\n def test_cleanup(self):\n self.manager.cleanup()\n self.store.cleanup.assert_called_with()\n\n def test_disabled(self):\n with self.manager.disabled():\n assert not self.manager.enabled\n assert self.manager.enabled\n\n def test_disabled_with_exception(self):\n def testfunc():\n with self.manager.disabled():\n raise ValueError()\n\n assert_raises(ValueError, testfunc)\n assert self.manager.enabled\n\n def test_disabled_nested(self):\n self.manager.enabled = False\n with self.manager.disabled():\n pass\n assert not self.manager.enabled\n\n\nclass TestStateChangeWatcher(TestCase):\n @setup\n def setup_watcher(self):\n self.watcher = StateChangeWatcher()\n self.state_manager = mock.create_autospec(PersistentStateManager)\n self.watcher.state_manager = self.state_manager\n\n def test_update_from_config_no_change(self):\n self.watcher.config = state_config = mock.Mock()\n assert not self.watcher.update_from_config(state_config)\n autospec_method(self.watcher.shutdown)\n assert_equal(self.watcher.state_manager, self.state_manager)\n assert not self.watcher.shutdown.mock_calls\n\n @mock.patch(\n \"tron.serialize.runstate.statemanager.PersistenceManagerFactory\",\n autospec=True,\n )\n def test_update_from_config_changed(self, mock_factory):\n state_config = mock.Mock()\n autospec_method(self.watcher.shutdown)\n assert self.watcher.update_from_config(state_config)\n assert_equal(self.watcher.config, state_config)\n self.watcher.shutdown.assert_called_with()\n assert_equal(\n self.watcher.state_manager,\n mock_factory.from_config.return_value,\n )\n mock_factory.from_config.assert_called_with(state_config)\n\n def test_save_job(self):\n mock_job = mock.Mock()\n self.watcher.save_job(mock_job)\n self.watcher.state_manager.save.assert_called_with(\n runstate.JOB_STATE,\n mock_job.name,\n mock_job.state_data,\n )\n\n def test_shutdown(self):\n self.watcher.shutdown()\n assert not self.watcher.state_manager.enabled\n self.watcher.state_manager.cleanup.assert_called_with()\n\n def test_disabled(self):\n context = self.watcher.disabled()\n assert_equal(self.watcher.state_manager.disabled.return_value, context)\n\n def test_restore(self):\n jobs = mock.Mock()\n self.watcher.restore(jobs)\n self.watcher.state_manager.restore.assert_called_with(jobs)\n\n def test_handler_mesos_change(self):\n self.watcher.handler(\n observable=MesosClusterRepository,\n event=None,\n )\n self.watcher.state_manager.save.assert_called_with(\n runstate.MESOS_STATE,\n MesosClusterRepository.name,\n MesosClusterRepository.state_data,\n )\n\n def test_handler_job_state_change(self):\n mock_job = mock.Mock(spec_set=Job)\n with mock.patch.object(self.watcher, \"save_job\") as mock_save_job:\n self.watcher.handler(\n observable=mock_job,\n event=Job.NOTIFY_STATE_CHANGE,\n )\n mock_save_job.assert_called_with(mock_job)\n\n def test_handler_job_new_run(self):\n mock_job = mock.Mock(spec_set=Job)\n mock_job_run = mock.Mock(spec_set=JobRun)\n with mock.patch.object(self.watcher, \"save_job\",) as mock_save_job, mock.patch.object(\n self.watcher,\n \"watch\",\n ) as mock_watch:\n # Error: No job run in event data, do nothing\n self.watcher.handler(\n observable=mock_job,\n event=Job.NOTIFY_NEW_RUN,\n )\n assert mock_watch.call_count == 0\n assert mock_save_job.call_count == 0\n\n # Correct case\n self.watcher.handler(\n observable=mock_job,\n event=Job.NOTIFY_NEW_RUN,\n event_data=mock_job_run,\n )\n mock_watch.assert_called_with(mock_job_run)\n assert mock_save_job.call_count == 0\n\n def test_handler_job_run_state_change(self):\n mock_job_run = mock.MagicMock(spec_set=JobRun)\n self.watcher.handler(\n observable=mock_job_run,\n event=JobRun.NOTIFY_STATE_CHANGED,\n )\n self.watcher.state_manager.save.assert_called_with(\n runstate.JOB_RUN_STATE,\n mock_job_run.name,\n mock_job_run.state_data,\n )\n\n def test_handler_job_run_removed(self):\n mock_job_run = mock.MagicMock(spec_set=JobRun)\n self.watcher.handler(\n observable=mock_job_run,\n event=JobRun.NOTIFY_REMOVED,\n )\n self.watcher.state_manager.delete.assert_called_with(\n runstate.JOB_RUN_STATE,\n mock_job_run.name,\n )\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/serialize/runstate/yamlstore_test.py", "content": "import os\nimport tempfile\n\nfrom testifycompat import assert_equal\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import teardown\nfrom testifycompat import TestCase\nfrom tron import yaml\nfrom tron.serialize.runstate import yamlstore\n\n\nclass TestYamlStateStore(TestCase):\n @setup\n def setup_store(self):\n self.filename = os.path.join(tempfile.gettempdir(), \"yaml_state\")\n self.store = yamlstore.YamlStateStore(self.filename)\n self.test_data = {\n \"one\": {\n \"a\": 1,\n },\n \"two\": {\n \"b\": 2,\n },\n \"three\": {\n \"c\": 3,\n },\n }\n\n @teardown\n def teardown_store(self):\n try:\n os.unlink(self.filename)\n except OSError:\n pass\n\n def test_restore(self):\n with open(self.filename, \"w\") as fh:\n yaml.dump(self.test_data, fh)\n\n keys = [yamlstore.YamlKey(\"one\", \"a\"), yamlstore.YamlKey(\"three\", \"c\")]\n state_data = self.store.restore(keys)\n assert_equal(self.store.buffer, self.test_data)\n\n expected = {keys[0]: 1, keys[1]: 3}\n assert_equal(expected, state_data)\n\n def test_restore_missing_type_key(self):\n with open(self.filename, \"w\") as fh:\n yaml.dump(self.test_data, fh)\n\n keys = [yamlstore.YamlKey(\"seven\", \"a\")]\n state_data = self.store.restore(keys)\n assert_equal(self.store.buffer, self.test_data)\n assert_equal({}, state_data)\n\n def test_restore_file_missing(self):\n state_data = self.store.restore([\"some\", \"keys\"])\n assert_equal(state_data, {})\n\n def test_save(self):\n expected = {\"one\": {\"five\": \"dataz\"}, \"two\": {\"seven\": \"stars\"}}\n\n key_value_pairs = [\n (yamlstore.YamlKey(\"one\", \"five\"), \"barz\"),\n ]\n # Save first\n self.store.save(key_value_pairs)\n\n # Save second\n key_value_pairs = [\n (yamlstore.YamlKey(\"two\", \"seven\"), \"stars\"),\n (yamlstore.YamlKey(\"one\", \"five\"), \"dataz\"),\n ]\n self.store.save(key_value_pairs)\n\n assert_equal(self.store.buffer, expected)\n with open(self.filename) as fh:\n actual = yaml.load(fh)\n assert_equal(actual, expected)\n\n def test_delete(self):\n expected = {\"state_a\": {\"five\": \"barz\"}}\n\n key_value_pairs = [\n (yamlstore.YamlKey(\"state_a\", \"five\"), \"barz\"),\n (yamlstore.YamlKey(\"state_c\", \"five\"), \"delete_all_c\"),\n (yamlstore.YamlKey(\"state_a\", \"six\"), \"delete_one_a\"),\n ]\n # Save first\n self.store.save(key_value_pairs)\n\n # Save second\n key_value_pairs = [\n (yamlstore.YamlKey(\"state_c\", \"five\"), None),\n (yamlstore.YamlKey(\"state_a\", \"six\"), None),\n ]\n self.store.save(key_value_pairs)\n\n assert_equal(self.store.buffer, expected)\n with open(self.filename) as fh:\n actual = yaml.load(fh)\n assert_equal(actual, expected)\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/ssh_test.py", "content": "from unittest import mock\n\nfrom twisted.python import failure\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_not_equal\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tests.testingutils import autospec_method\nfrom tron import ssh\n\n\nclass TestClientTransport(TestCase):\n @setup\n def setup_transport(self):\n self.username = \"username\"\n self.options = mock.Mock()\n self.expected_pub_key = mock.Mock()\n self.transport = ssh.ClientTransport(\n self.username,\n self.options,\n self.expected_pub_key,\n )\n\n def test_verifyHostKey_missing_pub_key(self):\n self.transport.expected_pub_key = None\n result = self.transport.verifyHostKey(mock.Mock(), mock.Mock())\n assert_equal(result.result, 1)\n\n @mock.patch(\"tron.ssh.keys\", autospec=True)\n def test_verifyHostKey_matching_pub_key(self, mock_keys):\n mock_keys.Key.fromString.return_value = self.expected_pub_key\n public_key = mock.Mock()\n result = self.transport.verifyHostKey(public_key, mock.Mock())\n assert_equal(result.result, 2)\n mock_keys.Key.fromString.assert_called_with(public_key)\n\n @mock.patch(\"tron.ssh.keys\", autospec=True)\n def test_verifyHostKey_mismatch_pub_key(self, _):\n public_key = mock.Mock()\n result = self.transport.verifyHostKey(public_key, mock.Mock())\n assert isinstance(result.result, failure.Failure)\n\n def test_connnectionSecure(self):\n self.transport.connection_defer = mock.Mock()\n autospec_method(self.transport.requestService)\n self.transport.connectionSecure()\n conn = self.transport.connection_defer.mock_calls[0][1][0]\n assert isinstance(conn, ssh.ClientConnection)\n auth_service = self.transport.requestService.mock_calls[0][1][0]\n assert isinstance(auth_service, ssh.NoPasswordAuthClient)\n\n\nclass TestSSHAuthOptions(TestCase):\n def test_from_config_none(self):\n ssh_conf = mock.Mock(agent=False, identities=[])\n ssh_options = ssh.SSHAuthOptions.from_config(ssh_conf)\n assert_equal(ssh_options[\"noagent\"], True)\n assert_equal(ssh_options.identitys, [])\n\n def test_from_config_both(self):\n identities = [\"one\", \"two\"]\n ssh_conf = mock.Mock(agent=True, identities=identities)\n ssh_options = ssh.SSHAuthOptions.from_config(ssh_conf)\n assert_equal(ssh_options[\"noagent\"], False)\n assert_equal(ssh_options.identitys, identities)\n\n def test__eq__true(self):\n config = mock.Mock(agent=True, identities=[\"one\", \"two\"])\n assert_equal(\n ssh.SSHAuthOptions.from_config(config),\n ssh.SSHAuthOptions.from_config(config),\n )\n\n def test__eq__false(self):\n config = mock.Mock(agent=True, identities=[\"one\", \"two\"])\n second_config = mock.Mock(agent=True, identities=[\"two\"])\n assert_not_equal(\n ssh.SSHAuthOptions.from_config(config),\n ssh.SSHAuthOptions.from_config(second_config),\n )\n" }, { "path": "tests/test_id_rsa", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAz+pjFOAHRLcQc6X51SyysJurwBTben3OWCG46CB81faxTVrC\ngcVEM3HCHz5MU8jI0Wb+DK9AXU229yQ8OCRPt3CrzxyI031ZrzuagVNRb/hmiBN+\n+SNmIFQl97/bqht5DXKykUJQmP31crLz3+G0rGXONeJjZAqFFIA1NfMMAnRZMRMo\n417Xf/p3yYXCV0AbWqbMWYA8aQKWFs4EY36vOzUJyftJI1cGttvskcCd3dce4DpR\n8JEnI/rkRKOR19eCLCtftV6CmC3igLoSF1JDeQgkVZRdVG/pcwfV1a/SEAlxAO0e\nRcZccMVoPi/Ans2CxIeukQ6ThFyRMcaD+iku6QIBIwKCAQBw3lMLVQtCj0Nx+wP3\nYWhRPpBvlktCfs8Z5muxNjUjsc32yt6eN+Mx3qs1iDgQOcwZ53P4QeEctSjPTi9R\nrU/YnEACt7f9x7RXz+Yo8rcuJ8KhpC78Rmqj1ei5sUtcWA6DpKoUVzMROWf8b8Y3\ntQpO9W/xXaOrVir8gBzixcSw3xgBiobkxWeiwCrD8LsPQ7qrP27nf252wrIa8BJx\n7qFrRSa05I6Hinj3jMwXUiUPVXBG5m+t8Z4c9WITvnr91uwHuHZc9TV6HedqznmA\n0n6iq2rPHPj4D0gWZFKvTFXN06KD+ldAiyWWGg3nob7gzFkGZkbAk2KLUN7PLuv3\n0KRPAoGBAPg8DkyznKNVgenaKt5Jc9YdLCVd4uvvjuTgmBiezvE4iwVRkIm6Q0jq\nFLgN4c4fUd10Fa4QPwIFXQBK5UYTwoklvDttZ+LnwY5SEzGeaXHdi8suOPlcDvDp\nL5Fpbi6wGJk04MXcApIwLIQCQp+pJeIHbKoq7c2ABMuqp+QPbUwrAoGBANZrcXfo\np2fIxbhrdwS7yMT0jInX6HeDvDM6e67X3TZYAjZuj9PNtlg6qOT6mFWUzOvcEFzj\nRDfA2d53DpGfpALdQ8OBxyI7QrdcbeW0Br/pGCicW+ovbT3Z3Bfl+GOhav1EuHRR\nL0GlW2xN3h+oZRexUSPAA80ep9o/XGaAjGM7AoGBAOoMvQZ9dm4da9x9PlyOZeci\n0doWsWIcYihBeXZMlzs1T+BxeaZtymlRu8N6zZZ1TS/ietdRJXbvHSwpW9RbxgxI\nJoEso8dPipwhf8+yne8EFhdXd4wGVzrqfU6WmxYTv2vhZjblYYKFMUlD9ax66TQy\n4sxUXI6OpW+SRoaSM9oZAoGBAKVo0+AogSQtKtAYYyDoojGJc7rLIQu9ZUwXLDZs\nAmvAPDiebvPZNOT6DULtM6+77ooQKeFBmwZwMwqznYZH840uWNil8WOMzREbariD\nkC2lL+TQZCmvjslQSrNZolt8hbwQcQlF8UFFDAMXf3eB55XvL/cB1wvzE8Wel7zJ\nkN7VAoGBAIISjKvjU3VpKNX9CEQ7V6eb7OuUKo3gUTCMaunWwpVYZ4pR5UiOtib+\nlcDpTQybKQOqSgHKQ13L/7Nu0GY9ILlXhfNhRlflSNUNaGcwMykxm0tzh3TsQ3vv\n8cQYV+W+24bwK39tSzl+n/nnuDdly7aTS2nDrhwWIsL45DU1QXK8\n-----END RSA PRIVATE KEY-----\n" }, { "path": "tests/test_id_rsa.pub", "content": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAz+pjFOAHRLcQc6X51SyysJurwBTben3OWCG46CB81faxTVrCgcVEM3HCHz5MU8jI0Wb+DK9AXU229yQ8OCRPt3CrzxyI031ZrzuagVNRb/hmiBN++SNmIFQl97/bqht5DXKykUJQmP31crLz3+G0rGXONeJjZAqFFIA1NfMMAnRZMRMo417Xf/p3yYXCV0AbWqbMWYA8aQKWFs4EY36vOzUJyftJI1cGttvskcCd3dce4DpR8JEnI/rkRKOR19eCLCtftV6CmC3igLoSF1JDeQgkVZRdVG/pcwfV1a/SEAlxAO0eRcZccMVoPi/Ans2CxIeukQ6ThFyRMcaD+iku6Q== rhettg@devvm1\n" }, { "path": "tests/testingutils.py", "content": "import functools\nimport logging\nimport time\nfrom unittest import mock\n\nfrom testifycompat import class_setup\nfrom testifycompat import class_teardown\nfrom testifycompat import setup\nfrom testifycompat import teardown\nfrom testifycompat import TestCase\nfrom tron.utils import timeutils\n\nlog = logging.getLogger(__name__)\n\n# TODO: remove when replaced with tron.eventloop\n\n\nclass MockReactorTestCase(TestCase):\n \"\"\"Patch the reactor to a MockReactor.\"\"\"\n\n # Override this in subclasses\n module_to_mock = None\n\n @class_setup\n def class_setup_patched_reactor(self):\n msg = \"%s must set a module_to_mock field\" % self.__class__\n assert self.module_to_mock, msg\n self.old_reactor = getattr(self.module_to_mock, \"reactor\")\n\n @class_teardown\n def teardown_patched_reactor(self):\n setattr(self.module_to_mock, \"reactor\", self.old_reactor)\n\n @setup\n def setup_mock_reactor(self):\n self.reactor = mock.MagicMock()\n setattr(self.module_to_mock, \"reactor\", self.reactor)\n\n\n# TODO: remove\nclass MockTimeTestCase(TestCase):\n\n now = None\n\n @setup\n def setup_current_time(self):\n assert self.now, \"%s must set a now field\" % self.__class__\n self.old_current_time = timeutils.current_time\n timeutils.current_time = lambda tz=None: self.now\n\n @teardown\n def teardown_current_time(self):\n timeutils.current_time = self.old_current_time\n # Reset 'now' back to what was set on the class because some test may\n # have changed it\n self.now = self.__class__.now\n\n\ndef retry(max_tries=3, delay=0.1, exceptions=(KeyError, IndexError)):\n \"\"\"A function decorator for re-trying an operation. Useful for MongoDB\n which is only eventually consistent.\n \"\"\"\n\n def wrapper(f):\n @functools.wraps(f)\n def wrap(*args, **kwargs):\n for _ in range(max_tries):\n try:\n return f(*args, **kwargs)\n except exceptions:\n time.sleep(delay)\n raise\n\n return wrap\n\n return wrapper\n\n\ndef autospec_method(method, *args, **kwargs):\n \"\"\"create an autospec for an instance method.\"\"\"\n mocked_method = mock.create_autospec(method, *args, **kwargs)\n setattr(method.__self__, method.__name__, mocked_method)\n" }, { "path": "tests/tools/sync_tron_state_from_k8s_test.py", "content": "from unittest import mock\n\nimport pytest\nfrom kubernetes.client import V1ObjectMeta\nfrom kubernetes.client import V1Pod\nfrom kubernetes.client import V1PodStatus\n\nfrom tools.sync_tron_state_from_k8s import get_matching_pod\nfrom tools.sync_tron_state_from_k8s import get_tron_state_from_api\nfrom tools.sync_tron_state_from_k8s import update_tron_from_pods\n\n\ndef create_mock_pod(name: str, phase: str, labels: dict[str, str], creation_timestamp: str):\n metadata = V1ObjectMeta(name=name, creation_timestamp=creation_timestamp, labels=labels)\n status = V1PodStatus(phase=phase)\n return V1Pod(metadata=metadata, status=status)\n\n\nclass TestSyncTronStateFromK8s:\n @pytest.fixture(autouse=True)\n def setup_test_data(self):\n self.pods = {\n p.metadata.name: p\n for p in [\n create_mock_pod(\n \"service.job.2.action\",\n \"Succeeded\",\n {\n \"paasta.yelp.com/service\": \"service\",\n \"paasta.yelp.com/instance\": \"job.action\",\n \"tron.yelp.com/run_num\": \"2\",\n },\n \"2024-01-01T00:00:00\",\n ),\n create_mock_pod(\n \"service.job.3.action-nomatch\",\n \"Failed\",\n {\n \"paasta.yelp.com/service\": \"service\",\n \"paasta.yelp.com/instance\": \"job.action\",\n \"tron.yelp.com/run_num\": \"3\",\n },\n \"2024-01-01T00:00:00\",\n ),\n create_mock_pod(\n \"service.job.4.action-nomatch\",\n \"Failed\",\n {\n \"paasta.yelp.com/service\": \"service\",\n \"paasta.yelp.com/instance\": \"job.action\",\n \"tron.yelp.com/run_num\": \"4\",\n },\n \"2024-01-01T00:00:00\",\n ),\n create_mock_pod(\n \"service.job.4.action-nomatch-retry2\",\n \"Succeeded\",\n {\n \"paasta.yelp.com/service\": \"service\",\n \"paasta.yelp.com/instance\": \"job.action\",\n \"tron.yelp.com/run_num\": \"4\",\n },\n \"2024-01-01T01:00:00\",\n ),\n create_mock_pod(\n \"service.job2.10.action\",\n \"Failed\",\n {\n \"paasta.yelp.com/service\": \"service\",\n \"paasta.yelp.com/instance\": \"job2.action\",\n \"tron.yelp.com/run_num\": \"10\",\n },\n \"2024-01-01T01:00:00\",\n ),\n create_mock_pod(\n \"service.job2.10.action\",\n \"Running\",\n {\n \"paasta.yelp.com/service\": \"service\",\n \"paasta.yelp.com/instance\": \"job2.action\",\n \"tron.yelp.com/run_num\": \"10\",\n },\n \"2024-01-01T01:05:00\",\n ),\n create_mock_pod(\n # Technically this pod name would not actually exist\n \"service.job_with_an_extremely_extremely_extremely_extremely_extremely_long_name.10.action\",\n \"Succeeded\",\n {\n \"paasta.yelp.com/service\": \"service\",\n # If PaaSTA's setup_tron_namespace changes how we create these labels, this test will need updating\n \"paasta.yelp.com/instance\": \"job_with_an_extremely_extremely_extremely_extremely_extrem-26i4\",\n \"tron.yelp.com/run_num\": \"10\",\n },\n \"2024-01-01T01:05:00\",\n ),\n ]\n }\n\n # 1 matching pod by labels\n # 2 matching pod by labels\n # no matching pod\n # test matching by hashed instance name\n @pytest.mark.parametrize(\n \"job_name,run_num,expected_pod_name\",\n [\n (\"service.job\", \"3\", \"service.job.3.action-nomatch\"),\n (\"service.job\", \"4\", \"service.job.4.action-nomatch-retry2\"),\n (\"service.job2\", \"10\", None),\n (\"service2.job\", \"1\", None),\n (\n \"service.job_with_an_extremely_extremely_extremely_extremely_extremely_long_name\",\n \"10\",\n \"service.job_with_an_extremely_extremely_extremely_extremely_extremely_long_name.10.action\",\n ),\n ],\n )\n def test_get_matching_pod(self, job_name, run_num, expected_pod_name):\n test_action_run = {\"action_name\": \"action\", \"job_name\": f\"{job_name}\", \"run_num\": run_num}\n matching_pod = get_matching_pod(test_action_run, self.pods)\n assert matching_pod == self.pods.get(expected_pod_name)\n\n # verify we send correct num_runs\n # verify we are sending request for jobs + one for each job\n @mock.patch(\"tools.sync_tron_state_from_k8s.get_client_config\", autospec=True)\n @mock.patch(\"tools.sync_tron_state_from_k8s.Client\", autospec=True)\n def test_get_tron_state_from_api(self, mock_client, mock_get_client_config):\n mock_client.return_value = mock.Mock()\n mock_client.return_value.jobs.return_value = [{\"url\": \"/uri\", \"name\": \"some job\"}]\n mock_client.return_value.job.return_value = {\"runs\": []}\n mock_get_client_config.return_value = {\"server\": \"https://localhost:8888\"}\n get_tron_state_from_api(None, num_runs=10)\n\n mock_client.assert_called_with(\"https://localhost:8888\")\n mock_client.return_value.jobs.assert_called_with(\n include_job_runs=False, include_action_runs=False, include_action_graph=False, include_node_pool=False\n )\n\n mock_client.return_value.job.assert_called_with(\"/api/uri\", include_action_runs=True, count=10)\n\n @mock.patch(\"tools.sync_tron_state_from_k8s.subprocess.run\", autospec=True)\n def test_update_tron(self, mock_subprocess_run):\n # sorry for the blob of test data\n tron_state = [\n {\n \"name\": \"service.job\",\n \"runs\": [\n {\n \"runs\": [\n {\n \"id\": \"service.job.2.action\",\n \"action_name\": \"action\",\n \"run_num\": \"2\",\n \"job_name\": \"service.job\",\n \"state\": \"unknown\",\n }\n ]\n },\n {\n \"runs\": [\n {\n \"id\": \"service.job.3.action\",\n \"action_name\": \"action\",\n \"run_num\": \"3\",\n \"job_name\": \"service.job\",\n \"state\": \"running\",\n },\n {\n \"id\": \"service.job.3.action2\",\n \"action_name\": \"action2\",\n \"run_num\": \"3\",\n \"job_name\": \"service.job\",\n \"state\": \"running\",\n },\n ]\n },\n {\n \"runs\": [\n {\n \"id\": \"service.job.4.action\",\n \"action_name\": \"action\",\n \"run_num\": \"4\",\n \"job_name\": \"service.job\",\n \"state\": \"starting\",\n }\n ]\n },\n {\n \"runs\": [\n {\n \"id\": \"service.job.5.action\",\n \"action_name\": \"action\",\n \"run_num\": \"5\",\n \"job_name\": \"service.job\",\n \"state\": \"starting\",\n }\n ]\n },\n ],\n },\n {\n \"name\": \"service.job2\",\n \"runs\": [\n {\n \"runs\": [\n {\n \"id\": \"service.job2.10.action\",\n \"action_name\": \"action\",\n \"run_num\": \"10\",\n \"job_name\": \"service.job2\",\n \"state\": \"succeeded\",\n },\n ]\n },\n ],\n },\n ]\n\n good_subprocess_run = mock.Mock(returncode=0)\n bad_subprocess_run = mock.Mock(returncode=1)\n\n expected_calls = [\n mock.call([\"tronctl\", \"success\", \"service.job.2.action\"], capture_output=True, text=True),\n mock.call([\"tronctl\", \"fail\", \"service.job.3.action\"], capture_output=True, text=True),\n mock.call([\"tronctl\", \"success\", \"service.job.4.action\"], capture_output=True, text=True),\n ]\n mock_subprocess_run.return_value = good_subprocess_run\n\n result = update_tron_from_pods(tron_state, self.pods, tronctl_wrapper=\"tronctl\", do_work=True)\n\n assert result[\"updated\"] == [\"service.job.2.action\", \"service.job.3.action\", \"service.job.4.action\"]\n assert result[\"error\"] == []\n mock_subprocess_run.assert_has_calls(expected_calls, any_order=True)\n\n mock_subprocess_run.return_value = bad_subprocess_run\n result = update_tron_from_pods(tron_state, self.pods, tronctl_wrapper=\"tronctl\", do_work=True)\n assert result[\"error\"] == [\"service.job.2.action\", \"service.job.3.action\", \"service.job.4.action\"]\n" }, { "path": "tests/trond_test.py", "content": "import datetime\nimport textwrap\nfrom subprocess import CalledProcessError\nfrom textwrap import dedent\n\nimport pytest\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_gt\nfrom tests import sandbox\nfrom tron.core import actionrun\n\nBASIC_CONFIG = \"\"\"\nssh_options:\n agent: true\n\nnodes:\n - name: local\n hostname: 'localhost'\n\nstate_persistence:\n name: \"state_data.shelve\"\n store_type: shelve\n\n\"\"\"\n\nSINGLE_ECHO_CONFIG = (\n BASIC_CONFIG\n + \"\"\"\njobs:\n - name: \"echo_job\"\n node: local\n schedule: \"cron 0 * * * *\"\n actions:\n - name: \"echo_action\"\n command: \"echo 'Echo!'\" \"\"\"\n)\n\nDOUBLE_ECHO_CONFIG = (\n SINGLE_ECHO_CONFIG\n + \"\"\"\n - name: \"another_echo_action\"\n command: \"echo 'Today is %(shortdate)s, which is the same\n as %(year)s-%(month)s-%(day)s' && false\" \"\"\"\n)\n\nALT_NAMESPACED_ECHO_CONFIG = \"\"\"\njobs:\n - name: \"echo_job\"\n node: local\n schedule: \"cron 0 * * * *\"\n actions:\n - name: \"echo_action\"\n command: \"echo 'Echo!'\" \"\"\"\n\nTOUCH_CLEANUP_FMT = \"\"\"\n cleanup_action:\n command: \"echo 'at last'\"\n\"\"\"\n\n\n@pytest.mark.skip(reason=\"We don't have a setup for sandbox tests yet\")\nclass TrondEndToEndTestCase(sandbox.SandboxTestCase):\n def test_end_to_end_basic(self):\n self.start_with_config(SINGLE_ECHO_CONFIG)\n client = self.sandbox.client\n\n assert_equal(\n self.client.config(\"MASTER\")[\"config\"],\n SINGLE_ECHO_CONFIG,\n )\n\n # reconfigure and confirm results\n second_config = DOUBLE_ECHO_CONFIG + TOUCH_CLEANUP_FMT\n self.sandbox.tronfig(second_config)\n assert_equal(client.config(\"MASTER\")[\"config\"], second_config)\n\n # reconfigure, by uploading a third configuration\n self.sandbox.tronfig(ALT_NAMESPACED_ECHO_CONFIG, name=\"ohce\")\n self.sandbox.client.home()\n\n # run the job and check its output\n echo_job_name = \"MASTER.echo_job\"\n job_url = client.get_url(echo_job_name)\n action_url = client.get_url(\"MASTER.echo_job.1.echo_action\")\n\n self.sandbox.tronctl(\"start\", echo_job_name)\n\n def wait_on_cleanup():\n return (\n len(client.job(job_url)[\"runs\"]) >= 2\n and client.action_runs(action_url)[\"state\"] == actionrun.ActionRun.SUCCEEDED\n )\n\n sandbox.wait_on_sandbox(wait_on_cleanup)\n\n echo_action_run = client.action_runs(action_url)\n another_action_url = client.get_url(\n \"MASTER.echo_job.1.another_echo_action\",\n )\n other_act_run = client.action_runs(another_action_url)\n assert_equal(\n echo_action_run[\"state\"],\n actionrun.ActionRun.SUCCEEDED,\n )\n assert_equal(echo_action_run[\"stdout\"], [\"Echo!\"])\n assert_equal(\n other_act_run[\"state\"],\n actionrun.ActionRun.FAILED,\n )\n\n now = datetime.datetime.now()\n stdout = now.strftime(\n \"Today is %Y-%m-%d, which is the same as %Y-%m-%d\",\n )\n assert_equal(other_act_run[\"stdout\"], [stdout])\n\n job_runs_url = client.get_url(\"%s.1\" % echo_job_name)\n assert_equal(\n client.job_runs(job_runs_url)[\"state\"],\n actionrun.ActionRun.FAILED,\n )\n\n def test_node_reconfig(self):\n job_config = dedent(\n \"\"\"\n jobs:\n - name: a_job\n node: local\n schedule: \"cron * * * * *\"\n actions:\n - name: first_action\n command: \"echo something\"\n \"\"\"\n )\n second_config = (\n dedent(\n \"\"\"\n ssh_options:\n agent: true\n\n nodes:\n - name: local\n hostname: '127.0.0.1'\n\n state_persistence:\n name: \"state_data.shelve\"\n store_type: shelve\n\n \"\"\"\n )\n + job_config\n )\n self.start_with_config(BASIC_CONFIG + job_config)\n\n job_url = self.client.get_url(\"MASTER.a_job.0\")\n sandbox.wait_on_state(\n self.client.job_runs,\n job_url,\n actionrun.ActionRun.SUCCEEDED,\n )\n\n self.sandbox.tronfig(second_config)\n\n job_url = self.client.get_url(\"MASTER.a_job\")\n\n def wait_on_next_run():\n last_run = self.client.job(job_url)[\"runs\"][0]\n return last_run[\"node\"][\"hostname\"] == \"127.0.0.1\"\n\n sandbox.wait_on_sandbox(wait_on_next_run)\n\n\n@pytest.mark.skip(reason=\"We don't have a setup for sandbox tests yet\")\nclass TronCommandsTestCase(sandbox.SandboxTestCase):\n def test_tronview(self):\n self.start_with_config(SINGLE_ECHO_CONFIG)\n expected = \"\"\"\\nServices:\\nNo Services\\n\\n\\nJobs:\n Name State Scheduler Last Success\n MASTER.echo_job enabled cron 1:00:00 None\n \"\"\"\n\n def remove_line_space(s):\n return [line.replace(\" \", \"\") for line in s.split(\"\\n\")]\n\n actual = self.sandbox.tronview()[0]\n assert_equal(remove_line_space(actual), remove_line_space(expected))\n\n def test_tronctl_with_job(self):\n self.start_with_config(SINGLE_ECHO_CONFIG + TOUCH_CLEANUP_FMT)\n job_name = \"MASTER.echo_job\"\n job_url = self.client.get_url(job_name)\n self.sandbox.tronctl(\"start\", job_name)\n\n cleanup_url = self.client.get_url(\"MASTER.echo_job.1.cleanup\")\n sandbox.wait_on_state(\n self.client.action_runs,\n cleanup_url,\n actionrun.ActionRun.SUCCEEDED,\n )\n\n action_run_url = self.client.get_url(\"MASTER.echo_job.1.echo_action\")\n assert_equal(\n self.client.action_runs(action_run_url)[\"state\"],\n actionrun.ActionRun.SUCCEEDED,\n )\n\n job_run_url = self.client.get_url(\"MASTER.echo_job.1\")\n assert_equal(\n self.client.job_runs(job_run_url)[\"state\"],\n actionrun.ActionRun.SUCCEEDED,\n )\n\n assert_equal(self.client.job(job_url)[\"status\"], \"enabled\")\n self.sandbox.tronctl(\"disable\", job_name)\n sandbox.wait_on_state(self.client.job, job_url, \"disabled\", \"status\")\n\n def test_tronfig(self):\n self.start_with_config(SINGLE_ECHO_CONFIG)\n stdout, stderr = self.sandbox.tronfig()\n assert_equal(stdout.rstrip(), SINGLE_ECHO_CONFIG.rstrip())\n\n def test_tronfig_failure(self):\n self.start_with_config(SINGLE_ECHO_CONFIG)\n bad_config = \"this is not valid: yaml: is it?\"\n\n def test_return_code(exc):\n assert_equal(exc.returncode, 1)\n\n with pytest.raises(CalledProcessError):\n test_return_code(self.sandbox.tronfig, bad_config)\n\n\n@pytest.mark.skip(reason=\"We don't have a setup for sandbox tests yet\")\nclass JobEndToEndTestCase(sandbox.SandboxTestCase):\n def test_cleanup_on_failure(self):\n config = (\n BASIC_CONFIG\n + dedent(\n \"\"\"\n jobs:\n - name: \"failjob\"\n node: local\n schedule: \"daily 04:20\"\n actions:\n - name: \"failaction\"\n command: \"failplz\"\n \"\"\"\n )\n + TOUCH_CLEANUP_FMT\n )\n self.start_with_config(config)\n\n action_run_url = self.client.get_url(\"MASTER.failjob.0.failaction\")\n sandbox.wait_on_state(\n self.client.action_runs,\n action_run_url,\n actionrun.ActionRun.FAILED,\n )\n\n action_run_url = self.client.get_url(\"MASTER.failjob.1.cleanup\")\n sandbox.wait_on_state(\n self.client.action_runs,\n action_run_url,\n actionrun.ActionRun.SUCCEEDED,\n )\n job_runs = self.client.job(\n self.client.get_url(\"MASTER.failjob\"),\n )[\"runs\"]\n assert_gt(len(job_runs), 1)\n\n def test_skip_failed_actions(self):\n config = BASIC_CONFIG + dedent(\n \"\"\"\n jobs:\n - name: \"multi_step_job\"\n node: local\n schedule: \"daily 04:20\"\n actions:\n - name: \"broken\"\n command: \"failingcommand\"\n - name: \"works\"\n command: \"echo ok\"\n requires: [broken]\n \"\"\"\n )\n self.start_with_config(config)\n action_run_url = self.client.get_url(\"MASTER.multi_step_job.0.broken\")\n waiter = sandbox.build_waiter_func(\n self.client.action_runs,\n action_run_url,\n )\n\n waiter(actionrun.ActionRun.FAILED)\n self.sandbox.tronctl(\"skip\", \"MASTER.multi_step_job.0.broken\")\n waiter(actionrun.ActionRun.SKIPPED)\n\n action_run_url = self.client.get_url(\"MASTER.multi_step_job.0.works\")\n sandbox.wait_on_state(\n self.client.action_runs,\n action_run_url,\n actionrun.ActionRun.SUCCEEDED,\n )\n\n job_run_url = self.client.get_url(\"MASTER.multi_step_job.0\")\n sandbox.wait_on_state(\n self.client.job_runs,\n job_run_url,\n actionrun.ActionRun.SUCCEEDED,\n )\n\n def test_failure_on_multi_step_job_doesnt_wedge_tron(self):\n config = BASIC_CONFIG + dedent(\n \"\"\"\n jobs:\n - name: \"random_failure_job\"\n node: local\n queueing: true\n schedule: \"daily 04:20\"\n actions:\n - name: \"fa\"\n command: \"sleep 0.1; failplz\"\n - name: \"sa\"\n command: \"echo 'you will never see this'\"\n requires: [fa]\n \"\"\"\n )\n self.start_with_config(config)\n job_url = self.client.get_url(\"MASTER.random_failure_job\")\n\n def wait_on_random_failure_job():\n return len(self.client.job(job_url)[\"runs\"]) >= 4\n\n sandbox.wait_on_sandbox(wait_on_random_failure_job)\n\n job_runs = self.client.job(job_url)[\"runs\"]\n expected = [actionrun.ActionRun.FAILED for _ in range(3)]\n assert_equal([run[\"state\"] for run in job_runs[-3:]], expected)\n\n def test_cancel_schedules_a_new_run(self):\n config = BASIC_CONFIG + dedent(\n \"\"\"\n jobs:\n - name: \"a_job\"\n node: local\n schedule: \"daily 05:00:00\"\n actions:\n - name: \"first_action\"\n command: \"echo OK\"\n \"\"\"\n )\n self.start_with_config(config)\n job_name = \"MASTER.a_job\"\n job_url = self.client.get_url(job_name)\n\n self.sandbox.tronctl(\"cancel\", \"%s.0\" % job_name)\n\n def wait_on_cancel():\n return len(self.client.job(job_url)[\"runs\"]) == 2\n\n sandbox.wait_on_sandbox(wait_on_cancel)\n\n run_states = [run[\"state\"] for run in self.client.job(job_url)[\"runs\"]]\n expected = [\n actionrun.ActionRun.SCHEDULED,\n actionrun.ActionRun.CANCELLED,\n ]\n assert_equal(run_states, expected)\n\n def test_job_queueing_false_with_overlap(self):\n \"\"\"Test that a job that has queueing false properly cancels an\n overlapping job run.\n \"\"\"\n config = BASIC_CONFIG + dedent(\n \"\"\"\n jobs:\n - name: \"cancel_overlap\"\n schedule: \"cron * * * * *\"\n queueing: False\n node: local\n actions:\n - name: \"do_something\"\n command: \"sleep 3s\"\n - name: \"do_other\"\n command: \"sleep 3s\"\n cleanup_action:\n command: \"echo done\"\n \"\"\"\n )\n self.start_with_config(config)\n job_url = self.client.get_url(\"MASTER.cancel_overlap\")\n job_run_url = self.client.get_url(\"MASTER.cancel_overlap.1\")\n\n def wait_on_job_schedule():\n return len(self.client.job(job_url)[\"runs\"]) == 2\n\n sandbox.wait_on_sandbox(wait_on_job_schedule)\n\n sandbox.wait_on_state(\n self.client.job,\n job_run_url,\n actionrun.ActionRun.CANCELLED,\n )\n\n action_run_states = [action_run[\"state\"] for action_run in self.client.job_runs(job_run_url)[\"runs\"]]\n expected = [actionrun.ActionRun.CANCELLED for _ in range(len(action_run_states))]\n assert_equal(action_run_states, expected)\n\n def test_trond_restart_job_with_run_history(self):\n config = BASIC_CONFIG + textwrap.dedent(\n \"\"\"\n jobs:\n - name: fast_job\n node: local\n schedule: daily 04:20\n actions:\n - name: single_act\n command: \"sleep 20 && echo good\"\n \"\"\"\n )\n self.start_with_config(config)\n\n action_run_url = self.client.get_url(\"MASTER.fast_job.0.single_act\")\n sandbox.wait_on_state(\n self.client.action_runs,\n action_run_url,\n actionrun.ActionRun.RUNNING,\n )\n\n self.restart_trond()\n\n assert_equal(\n self.client.job_runs(action_run_url)[\"state\"],\n actionrun.ActionRun.UNKNOWN,\n )\n\n next_run_url = self.client.get_url(\"MASTER.fast_job.-1.single_act\")\n sandbox.wait_on_state(\n self.client.action_runs,\n next_run_url,\n actionrun.ActionRun.RUNNING,\n )\n\n def test_trond_restart_job_running_with_dependencies(self):\n config = BASIC_CONFIG + textwrap.dedent(\n \"\"\"\n jobs:\n - name: complex_job\n node: local\n schedule: cron * * * * *\n actions:\n - name: first_act\n command: sleep 20 && echo \"I'm waiting\"\n - name: following_act\n command: echo \"thing\"\n requires: ['first_act']\n - name: last_act\n command: echo foo\n requires: ['following_act']\n \"\"\"\n )\n self.start_with_config(config)\n job_name = \"MASTER.complex_job\"\n self.sandbox.tronctl(\"start\", job_name)\n\n action_run_url = self.client.get_url(\"MASTER.complex_job.1.first_act\")\n sandbox.wait_on_state(\n self.client.action_runs,\n action_run_url,\n actionrun.ActionRun.RUNNING,\n )\n\n self.restart_trond()\n\n assert_equal(\n self.client.job_runs(action_run_url)[\"state\"],\n actionrun.ActionRun.UNKNOWN,\n )\n\n for followup_action_run in (\"following_act\", \"last_act\"):\n url = self.client.get_url(\n f\"{job_name}.1.{followup_action_run}\",\n )\n assert_equal(\n self.client.action_runs(url)[\"state\"],\n actionrun.ActionRun.QUEUED,\n )\n" }, { "path": "tests/trondaemon_test.py", "content": "import os\nimport tempfile\nimport time\nfrom unittest import mock\n\nfrom testifycompat import setup\nfrom testifycompat import teardown\nfrom testifycompat import TestCase\nfrom tron.trondaemon import TronDaemon\n\n\nclass TronDaemonTestCase(TestCase):\n @setup\n @mock.patch(\"tron.trondaemon.setup_logging\", mock.Mock(), autospec=None)\n @mock.patch(\"signal.signal\", mock.Mock(), autospec=None)\n def setup(self):\n self.tmpdir = tempfile.TemporaryDirectory()\n trond_opts = mock.Mock()\n trond_opts.working_dir = self.tmpdir.name\n trond_opts.lock_file = os.path.join(self.tmpdir.name, \"lockfile\")\n self.trond = TronDaemon(trond_opts)\n\n @teardown\n def teardown(self):\n self.tmpdir.cleanup()\n\n @mock.patch(\"tron.trondaemon.setup_logging\", mock.Mock(), autospec=None)\n @mock.patch(\"signal.signal\", mock.Mock(), autospec=None)\n def test_init(self):\n daemon = TronDaemon.__new__(TronDaemon) # skip __init__\n options = mock.Mock()\n\n with mock.patch(\n \"tron.utils.flock\",\n autospec=True,\n ) as mock_flock:\n daemon.__init__(options)\n assert mock_flock.call_count == 0\n\n def test_run_uses_context(self):\n with mock.patch(\"tron.trondaemon.setup_logging\", mock.Mock(), autospec=None,), mock.patch(\n \"tron.trondaemon.no_daemon_context\",\n mock.Mock(),\n autospec=None,\n ) as ndc:\n ndc.return_value = mock.MagicMock()\n boot_time = time.time()\n ndc.return_value.__enter__.side_effect = RuntimeError()\n daemon = TronDaemon(mock.Mock())\n try:\n daemon.run(boot_time)\n except Exception:\n pass\n assert ndc.call_count == 1\n\n def test_run_manhole_new_manhole(self):\n with open(self.trond.manhole_sock, \"w+\"):\n pass\n\n with mock.patch(\n \"twisted.internet.reactor.listenUNIX\",\n autospec=True,\n ) as mock_listenUNIX:\n self.trond._run_manhole()\n\n assert mock_listenUNIX.call_count == 1\n # _run_manhole will remove the old manhole.sock but not recreate\n # it because we mocked out listenUNIX\n assert not os.path.exists(self.trond.manhole_sock)\n" }, { "path": "tests/utils/__init__.py", "content": "" }, { "path": "tests/utils/collections_test.py", "content": "from unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_in\nfrom testifycompat import assert_not_in\nfrom testifycompat import assert_raises\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tests.assertions import assert_mock_calls\nfrom tests.testingutils import autospec_method\nfrom tron.utils import collections\n\n\nclass TestMappingCollections(TestCase):\n @setup\n def setup_collection(self):\n self.name = \"some_name\"\n self.collection = collections.MappingCollection(self.name)\n\n def test_filter_by_name(self):\n autospec_method(self.collection.remove)\n self.collection.update(dict.fromkeys([\"c\", \"d\", \"e\"]))\n self.collection.filter_by_name([\"a\", \"c\"])\n expected = [mock.call(name) for name in [\"d\", \"e\"]]\n assert_mock_calls(expected, self.collection.remove.mock_calls)\n\n def test_remove_missing(self):\n assert_raises(ValueError, self.collection.remove, \"name\")\n\n def test_remove(self):\n name = \"the_name\"\n self.collection[name] = item = mock.Mock()\n self.collection.remove(name)\n assert_not_in(name, self.collection)\n item.disable.assert_called_with()\n\n def test_contains_item_false(self):\n mock_item, mock_func = mock.Mock(), mock.Mock()\n assert not self.collection.contains_item(mock_item, mock_func)\n assert not mock_func.mock_calls\n\n def test_contains_item_not_equal(self):\n mock_item, mock_func = mock.Mock(), mock.Mock()\n self.collection[mock_item.get_name()] = \"other item\"\n result = self.collection.contains_item(mock_item, mock_func)\n assert_equal(result, mock_func.return_value)\n mock_func.assert_called_with(mock_item)\n\n def test_contains_item_true(self):\n mock_item, mock_func = mock.Mock(), mock.Mock()\n self.collection[mock_item.get_name()] = mock_item\n assert self.collection.contains_item(mock_item, mock_func)\n\n def test_add_contains(self):\n autospec_method(self.collection.contains_item)\n item, update_func = mock.Mock(), mock.Mock()\n assert not self.collection.add(item, update_func)\n assert_not_in(item.get_name(), self.collection)\n\n def test_add_new(self):\n autospec_method(self.collection.contains_item, return_value=False)\n item, update_func = mock.Mock(), mock.Mock()\n assert self.collection.add(item, update_func)\n assert_in(item.get_name(), self.collection)\n\n def test_replace(self):\n autospec_method(self.collection.add)\n item = mock.Mock()\n self.collection.replace(item)\n self.collection.add.assert_called_with(\n item,\n self.collection.remove_item,\n )\n" }, { "path": "tests/utils/crontab_test.py", "content": "from unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import assert_raises\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tron.utils import crontab\n\n\nclass TestConvertPredefined(TestCase):\n def test_convert_predefined_valid(self):\n expected = crontab.PREDEFINED_SCHEDULE[\"@hourly\"]\n assert_equal(crontab.convert_predefined(\"@hourly\"), expected)\n\n def test_convert_predefined_invalid(self):\n assert_raises(ValueError, crontab.convert_predefined, \"@bogus\")\n\n def test_convert_predefined_none(self):\n line = \"something else\"\n assert_equal(crontab.convert_predefined(line), line)\n\n\nclass TestParseCrontab(TestCase):\n def test_parse_asterisk(self):\n line = \"* * * * *\"\n actual = crontab.parse_crontab(line)\n assert_equal(actual[\"minutes\"], None)\n assert_equal(actual[\"hours\"], None)\n assert_equal(actual[\"months\"], None)\n\n @mock.patch(\"tron.utils.crontab.MinuteFieldParser.parse\", autospec=True)\n @mock.patch(\"tron.utils.crontab.HourFieldParser.parse\", autospec=True)\n @mock.patch(\"tron.utils.crontab.MonthdayFieldParser.parse\", autospec=True)\n @mock.patch(\"tron.utils.crontab.MonthFieldParser.parse\", autospec=True)\n @mock.patch(\"tron.utils.crontab.WeekdayFieldParser.parse\", autospec=True)\n def test_parse(self, mock_dow, mock_month, mock_monthday, mock_hour, mock_min):\n line = \"* * * * *\"\n actual = crontab.parse_crontab(line)\n assert_equal(actual[\"minutes\"], mock_min.return_value)\n assert_equal(actual[\"hours\"], mock_hour.return_value)\n assert_equal(actual[\"monthdays\"], mock_monthday.return_value)\n assert_equal(actual[\"months\"], mock_month.return_value)\n assert_equal(actual[\"weekdays\"], mock_dow.return_value)\n\n def test_full_crontab_line(self):\n line = \"*/15 0 1,15 * 1-5\"\n expected = {\n \"minutes\": [0, 15, 30, 45],\n \"hours\": [0],\n \"monthdays\": [1, 15],\n \"months\": None,\n \"weekdays\": [1, 2, 3, 4, 5],\n \"ordinals\": None,\n }\n assert_equal(crontab.parse_crontab(line), expected)\n\n def test_full_crontab_line_with_last(self):\n line = \"0 0 L * *\"\n expected = {\n \"minutes\": [0],\n \"hours\": [0],\n \"monthdays\": [\"LAST\"],\n \"months\": None,\n \"weekdays\": None,\n \"ordinals\": None,\n }\n assert_equal(crontab.parse_crontab(line), expected)\n\n\nclass TestMinuteFieldParser(TestCase):\n @setup\n def setup_parser(self):\n self.parser = crontab.MinuteFieldParser()\n\n def test_validate_bounds(self):\n assert_equal(self.parser.validate_bounds(0), 0)\n assert_equal(self.parser.validate_bounds(59), 59)\n assert_raises(ValueError, self.parser.validate_bounds, 60)\n\n def test_get_values_asterisk(self):\n assert_equal(self.parser.get_values(\"*\"), list(range(0, 60)))\n\n def test_get_values_min_only(self):\n assert_equal(self.parser.get_values(\"4\"), [4])\n assert_equal(self.parser.get_values(\"33\"), [33])\n\n def test_get_values_with_step(self):\n assert_equal(self.parser.get_values(\"*/10\"), [0, 10, 20, 30, 40, 50])\n\n def test_get_values_with_step_and_range(self):\n assert_equal(self.parser.get_values(\"10-30/10\"), [10, 20, 30])\n\n def test_get_values_with_step_and_overflow_range(self):\n assert_equal(self.parser.get_values(\"30-0/10\"), [30, 40, 50, 0])\n\n def test_parse_with_groups(self):\n assert_equal(self.parser.parse(\"5,1,7,8,5\"), [1, 5, 7, 8])\n\n def test_parse_with_groups_and_ranges(self):\n expected = [0, 1, 11, 13, 15, 17, 19, 20, 21, 40]\n assert_equal(self.parser.parse(\"1,11-22/2,*/20\"), expected)\n\n\nclass TestMonthFieldParser(TestCase):\n @setup\n def setup_parser(self):\n self.parser = crontab.MonthFieldParser()\n\n def test_parse(self):\n expected = [1, 2, 3, 7, 12]\n assert_equal(self.parser.parse(\"DEC, Jan-Feb, jul, MaR\"), expected)\n\n\nclass TestWeekdayFieldParser(TestCase):\n @setup\n def setup_parser(self):\n self.parser = crontab.WeekdayFieldParser()\n\n def test_parser(self):\n expected = [0, 3, 5, 6]\n assert_equal(self.parser.parse(\"Sun, 3, FRI, SaT-Sun\"), expected)\n\n\nclass TestMonthdayFieldParser(TestCase):\n @setup\n def setup_parser(self):\n self.parser = crontab.MonthdayFieldParser()\n\n def test_parse_last(self):\n expected = [5, 6, \"LAST\"]\n assert_equal(self.parser.parse(\"5, 6, L\"), expected)\n\n\nclass TestComplexExpressions(TestCase):\n @setup\n def setup_parser(self):\n self.parser = crontab.MinuteFieldParser()\n\n def test_complex_expression(self):\n expected = [0, 10, 20, 30, 40, 50, 55]\n assert_equal(self.parser.parse(\"*/10,55\"), expected)\n\n\nclass TestInvalidInputs(TestCase):\n @setup\n def setup_parser(self):\n self.parser = crontab.MinuteFieldParser()\n\n def test_invalid_expression(self):\n with assert_raises(ValueError):\n self.parser.parse(\"61\")\n\n\nclass TestBoundaryValues(TestCase):\n @setup\n def setup_parser(self):\n self.parser = crontab.MinuteFieldParser()\n\n def test_boundary_values(self):\n assert_equal(self.parser.parse(\"0\"), [0])\n assert_equal(self.parser.parse(\"59\"), [59])\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/utils/logreader_test.py", "content": "import datetime\nfrom unittest import mock\n\nimport pytest\nimport yaml\n\nimport tron.utils.logreader\nfrom tron.utils.logreader import decompose_action_id\nfrom tron.utils.logreader import read_log_stream_for_action_run\n\ntry:\n from logreader.readers import S3LogsReader # noqa: F401\nexcept ImportError:\n pytest.skip(\"yelp logs readers not available, skipping tests\", allow_module_level=True)\n\n\n# used for an explicit patch of staticconf.read return value for an arbitrary namespace\ndef static_conf_patch(args):\n return lambda arg, namespace, default=None: args.get(arg)\n\n\ndef test_read_log_stream_for_action_run_not_available():\n with mock.patch(\"tron.utils.logreader.s3reader_available\", False):\n output = tron.utils.logreader.read_log_stream_for_action_run(\n \"namespace.job.1234.action\",\n component=\"stdout\",\n min_date=datetime.datetime.now(),\n max_date=datetime.datetime.now(),\n paasta_cluster=\"fake\",\n )\n assert \"unable to display logs\" in output[0]\n\n\ndef test_read_log_stream_for_action_run():\n with mock.patch(\n \"staticconf.read\",\n autospec=True,\n side_effect=static_conf_patch({\"logging.max_lines_to_display\": 1000}),\n ), mock.patch(\"tron.config.static_config.build_configuration_watcher\", autospec=True,), mock.patch(\n \"tron.config.static_config.load_yaml_file\",\n autospec=True,\n ), mock.patch(\n \"tron.utils.logreader.get_superregion\", autospec=True, return_value=\"fake\"\n ), mock.patch(\n \"tron.utils.logreader.S3LogsReader\", autospec=True\n ) as mock_s3_reader:\n\n mock_s3_reader.return_value.get_log_reader.return_value = iter(\n [\n \"\"\"{\n \"tron_run_number\": 1234,\n \"component\": \"stdout\",\n \"message\": \"line 1\",\n \"timestamp\": \"2021-01-02T18:10:09.169421619Z\",\n \"cluster\": \"fake\"\n }\"\"\",\n \"\"\"{\n \"tron_run_number\": 1234,\n \"component\": \"stdout\",\n \"message\": \"line 2\",\n \"timestamp\": \"2021-01-02T18:11:09.169421619Z\",\n \"cluster\": \"fake\"\n }\"\"\",\n \"\"\"{\n \"tron_run_number\": 1234,\n \"component\": \"stderr\",\n \"message\": \"line 3\",\n \"timestamp\": \"2021-01-02T18:12:09.169421619Z\",\n \"cluster\": \"fake\"\n }\"\"\",\n ]\n )\n\n output = read_log_stream_for_action_run(\n \"namespace.job.1234.action\",\n component=\"stdout\",\n min_date=datetime.datetime.now(),\n max_date=datetime.datetime.now(),\n paasta_cluster=\"fake\",\n )\n\n mock_s3_reader.return_value.get_log_reader.assert_called_once_with(\n log_name=\"stream_paasta_app_output_namespace_job__action\", start_datetime=mock.ANY, end_datetime=mock.ANY\n )\n assert output == [\"line 1\", \"line 2\"]\n\n\n@pytest.mark.parametrize(\n \"local_datetime, expected_datetime\",\n [\n (\n datetime.datetime(2024, 2, 29, 23, 59, 59, tzinfo=datetime.timezone(datetime.timedelta(hours=+3))),\n datetime.datetime(2024, 2, 29, 20, 59, 59, tzinfo=datetime.timezone.utc),\n ),\n (\n datetime.datetime(2024, 2, 29, 23, 59, 59, tzinfo=datetime.timezone(datetime.timedelta(hours=-3))),\n datetime.datetime(2024, 3, 1, 2, 59, 59, tzinfo=datetime.timezone.utc),\n ),\n ],\n)\ndef test_read_log_stream_for_action_run_tz(local_datetime, expected_datetime):\n with mock.patch(\n \"staticconf.read\",\n autospec=True,\n side_effect=static_conf_patch({\"logging.max_lines_to_display\": 1000}),\n ), mock.patch(\"tron.config.static_config.build_configuration_watcher\", autospec=True,), mock.patch(\n \"tron.config.static_config.load_yaml_file\",\n autospec=True,\n ), mock.patch(\n \"tron.utils.logreader.get_superregion\", autospec=True, return_value=\"fake\"\n ), mock.patch(\n \"tron.utils.logreader.S3LogsReader\", autospec=True\n ) as mock_s3_log_reader:\n\n read_log_stream_for_action_run(\n \"namespace.job.1234.action\",\n component=\"stdout\",\n min_date=local_datetime,\n max_date=local_datetime,\n paasta_cluster=\"fake\",\n )\n mock_s3_log_reader.return_value.get_log_reader.assert_called_once_with(\n log_name=mock.ANY, start_datetime=expected_datetime, end_datetime=expected_datetime\n )\n\n\ndef test_read_log_stream_for_action_run_for_long_output():\n # 1000 represents the number of lines that are expected to be\n # outputted by the test, which is similar to the logging.max_lines_to_display\n # in tron.yaml in srv-configs\n max_lines = 1000\n with mock.patch(\"tron.utils.logreader.get_superregion\", autospec=True, return_value=\"fake\",), mock.patch(\n \"tron.config.static_config.build_configuration_watcher\",\n autospec=True,\n ), mock.patch(\n \"staticconf.read\", autospec=True, side_effect=static_conf_patch({\"logging.max_lines_to_display\": 1000})\n ), mock.patch(\n \"tron.config.static_config.load_yaml_file\",\n autospec=True,\n ), mock.patch(\n \"tron.utils.logreader.S3LogsReader\", autospec=True\n ) as mock_s3_reader:\n\n with open(\"./tests/utils/shortOutputTest.txt\") as f:\n content_list = f.readlines()\n\n mock_s3_reader.return_value.get_log_reader.return_value = iter(content_list)\n\n output = read_log_stream_for_action_run(\n \"namespace.job.228.action\",\n component=\"stdout\",\n min_date=datetime.datetime.now(),\n max_date=datetime.datetime.now(),\n paasta_cluster=\"infrastage\",\n )\n\n mock_s3_reader.return_value.get_log_reader.assert_called_once_with(\n log_name=\"stream_paasta_app_output_namespace_job__action\", start_datetime=mock.ANY, end_datetime=mock.ANY\n )\n assert len(output) == max_lines + 1\n\n\ndef test_decompose_action_id_file_not_found():\n action_run_id = \"namespace.job.1234.action\"\n paasta_cluster = \"fake_cluster\"\n with mock.patch(\"builtins.open\", side_effect=FileNotFoundError):\n namespace, job_name, run_num, action = decompose_action_id(action_run_id, paasta_cluster)\n assert namespace == \"namespace\"\n assert job_name == \"job\"\n assert run_num == \"1234\"\n assert action == \"action\"\n\n\ndef test_decompose_action_id_yaml_error():\n action_run_id = \"namespace.job.1234.action\"\n paasta_cluster = \"fake_cluster\"\n with mock.patch(\"builtins.open\", mock.mock_open(read_data=\"invalid_yaml\")), mock.patch(\n \"yaml.safe_load\", side_effect=yaml.YAMLError\n ):\n namespace, job_name, run_num, action = decompose_action_id(action_run_id, paasta_cluster)\n assert namespace == \"namespace\"\n assert job_name == \"job\"\n assert run_num == \"1234\"\n assert action == \"action\"\n\n\ndef test_decompose_action_id_generic_error():\n action_run_id = \"namespace.job.1234.action\"\n paasta_cluster = \"fake_cluster\"\n with mock.patch(\"builtins.open\", mock.mock_open(read_data=\"some_data\")), mock.patch(\n \"yaml.safe_load\", side_effect=Exception\n ):\n namespace, job_name, run_num, action = decompose_action_id(action_run_id, paasta_cluster)\n assert namespace == \"namespace\"\n assert job_name == \"job\"\n assert run_num == \"1234\"\n assert action == \"action\"\n\n\ndef test_decompose_action_id_service_not_found():\n action_run_id = \"namespace.job.1234.action\"\n paasta_cluster = \"fake_cluster\"\n config_content = \"\"\"\n job:\n actions:\n action:\n command: \"sleep 10\"\n \"\"\"\n with mock.patch(\"builtins.open\", mock.mock_open(read_data=config_content)), mock.patch(\n \"yaml.safe_load\", return_value=yaml.safe_load(config_content)\n ):\n namespace, job_name, run_num, action = decompose_action_id(action_run_id, paasta_cluster)\n assert namespace == \"namespace\"\n assert job_name == \"job\"\n assert run_num == \"1234\"\n assert action == \"action\"\n" }, { "path": "tests/utils/observer_test.py", "content": "from unittest import mock\n\nfrom testifycompat import assert_equal\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tests.assertions import assert_length\nfrom tron.utils.observer import Observable\nfrom tron.utils.observer import Observer\n\n\nclass TestObservable(TestCase):\n @setup\n def setup_observer(self):\n self.obs = Observable()\n\n def test_attach(self):\n def func():\n return 1\n\n self.obs.attach(\"a\", func)\n assert_equal(len(self.obs._observers), 1)\n assert_equal(self.obs._observers[\"a\"], [func])\n\n def test_listen_seq(self):\n def func():\n return 1\n\n self.obs.attach([\"a\", \"b\"], func)\n assert_equal(len(self.obs._observers), 2)\n assert_equal(self.obs._observers[\"a\"], [func])\n assert_equal(self.obs._observers[\"b\"], [func])\n\n def test_notify(self):\n handler = mock.MagicMock()\n self.obs.attach([\"a\", \"b\"], handler)\n self.obs.notify(\"a\")\n assert_equal(len(handler.handler.mock_calls), 1)\n self.obs.notify(\"b\")\n assert_equal(len(handler.handler.mock_calls), 2)\n\n\nclass TestObserverClear(TestCase):\n @setup\n def setup_observer(self):\n self.obs = Observable()\n\n def func():\n return 1\n\n self.obs.attach(\"a\", func)\n self.obs.attach(\"b\", func)\n self.obs.attach(True, func)\n self.obs.attach([\"a\", \"b\"], func)\n\n def test_clear_listeners_all(self):\n self.obs.clear_observers()\n assert_equal(len(self.obs._observers), 0)\n\n def test_clear_listeners_some(self):\n self.obs.clear_observers(\"a\")\n assert_equal(len(self.obs._observers), 2)\n assert_equal(set(self.obs._observers.keys()), {True, \"b\"})\n\n def test_remove_observer_none(self):\n def observer():\n return 2\n\n self.obs.remove_observer(observer)\n assert_equal(set(self.obs._observers.keys()), {True, \"a\", \"b\"})\n assert_length(self.obs._observers[\"a\"], 2)\n assert_length(self.obs._observers[\"b\"], 2)\n assert_length(self.obs._observers[True], 1)\n\n def test_remove_observer(self):\n def observer():\n return 2\n\n self.obs.attach(\"a\", observer)\n self.obs.attach(\"c\", observer)\n self.obs.remove_observer(observer)\n assert_length(self.obs._observers[\"a\"], 2)\n assert_length(self.obs._observers[\"b\"], 2)\n assert_length(self.obs._observers[True], 1)\n assert_length(self.obs._observers[\"c\"], 0)\n\n\nclass MockObserver(Observer):\n def __init__(self, obs, event):\n self.obs = obs\n self.event = event\n self.watch(obs, event)\n self.has_watched = 0\n self.event_data = None\n\n def handler(self, obs, event, event_data):\n assert_equal(obs, self.obs)\n assert_equal(event, self.event)\n self.has_watched += 1\n self.event_data = event_data\n\n\nclass TestObserver(TestCase):\n @setup\n def setup_observer(self):\n self.obs = Observable()\n\n def test_watch(self):\n event = \"FIVE\"\n handler = MockObserver(self.obs, event)\n\n self.obs.notify(event)\n assert_equal(handler.has_watched, 1)\n assert handler.event_data is None\n self.obs.notify(\"other event\")\n assert_equal(handler.has_watched, 1)\n self.obs.notify(event, \"event_data\")\n assert_equal(handler.has_watched, 2)\n assert handler.event_data == \"event_data\"\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/utils/proxy_test.py", "content": "from testifycompat import assert_equal\nfrom testifycompat import assert_in\nfrom testifycompat import assert_raises\nfrom testifycompat import run\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tron.utils.proxy import AttributeProxy\nfrom tron.utils.proxy import CollectionProxy\n\n\nclass DummyTarget:\n def __init__(self, v):\n self.v = v\n\n def foo(self):\n return self.v\n\n @property\n def not_foo(self):\n return not self.v\n\n def equals(self, b, sometimes=False):\n if sometimes:\n return \"sometimes\"\n return self.v == b\n\n\nclass DummyObject:\n def __init__(self, proxy):\n self.proxy = proxy\n\n def __getattr__(self, item):\n return self.proxy.perform(item)\n\n\nclass TestCollectionProxy(TestCase):\n @setup\n def setup_proxy(self):\n self.target_list = [DummyTarget(1), DummyTarget(2), DummyTarget(0)]\n self.proxy = CollectionProxy(\n lambda: self.target_list,\n [\n (\"foo\", any, True),\n (\"not_foo\", all, False),\n (\"equals\", lambda a: list(a), True),\n ],\n )\n self.dummy = DummyObject(self.proxy)\n\n def test_add(self):\n self.proxy.add(\"foo\", any, True)\n assert_equal(self.proxy._defs[\"foo\"], (any, True))\n\n def test_perform(self):\n assert self.dummy.foo()\n assert not self.dummy.not_foo\n\n def test_perform_not_defined(self):\n assert_raises(AttributeError, self.dummy.proxy.perform, \"bar\")\n\n def test_perform_with_params(self):\n assert_equal(self.proxy.perform(\"equals\")(2), [False, True, False])\n sometimes = [\"sometimes\"] * 3\n assert_equal(\n self.proxy.perform(\"equals\")(3, sometimes=True),\n sometimes,\n )\n\n\nclass TestAttributeProxy(TestCase):\n @setup\n def setup_proxy(self):\n self.target = DummyTarget(1)\n self.proxy = AttributeProxy(self.target, [\"foo\", \"not_foo\"])\n self.dummy = DummyObject(self.proxy)\n\n def test_add(self):\n self.proxy.add(\"bar\")\n assert_in(\"bar\", self.proxy._attributes)\n\n def test_perform(self):\n assert_equal(self.dummy.foo(), 1)\n assert_equal(self.dummy.not_foo, False)\n\n def test_perform_not_defined(self):\n assert_raises(AttributeError, self.dummy.proxy.perform, \"zzz\")\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tests/utils/shortOutputTest.txt", "content": "{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1\",\"timestamp\":\"2022-08-12T13:30:04.686637906Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"5\",\"timestamp\":\"2022-08-12T13:30:04.686730242Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"2\",\"timestamp\":\"2022-08-12T13:30:04.686653695Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"3\",\"timestamp\":\"2022-08-12T13:30:04.68670797Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"6\",\"timestamp\":\"2022-08-12T13:30:04.686799509Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"4\",\"timestamp\":\"2022-08-12T13:30:04.686710726Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"7\",\"timestamp\":\"2022-08-12T13:30:04.68702548Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"8\",\"timestamp\":\"2022-08-12T13:30:04.687027597Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"9\",\"timestamp\":\"2022-08-12T13:30:04.687077736Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"10\",\"timestamp\":\"2022-08-12T13:30:04.687161226Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"11\",\"timestamp\":\"2022-08-12T13:30:04.687273728Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"12\",\"timestamp\":\"2022-08-12T13:30:04.687277071Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"13\",\"timestamp\":\"2022-08-12T13:30:04.687306184Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"14\",\"timestamp\":\"2022-08-12T13:30:04.687341593Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"15\",\"timestamp\":\"2022-08-12T13:30:04.687488029Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"16\",\"timestamp\":\"2022-08-12T13:30:04.687489495Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"17\",\"timestamp\":\"2022-08-12T13:30:04.687502175Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"18\",\"timestamp\":\"2022-08-12T13:30:04.687542498Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"21\",\"timestamp\":\"2022-08-12T13:30:04.687638635Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"19\",\"timestamp\":\"2022-08-12T13:30:04.687618198Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"22\",\"timestamp\":\"2022-08-12T13:30:04.687670676Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"20\",\"timestamp\":\"2022-08-12T13:30:04.687620336Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"23\",\"timestamp\":\"2022-08-12T13:30:04.687813739Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"24\",\"timestamp\":\"2022-08-12T13:30:04.687817306Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"25\",\"timestamp\":\"2022-08-12T13:30:04.687839161Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"26\",\"timestamp\":\"2022-08-12T13:30:04.687878011Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"27\",\"timestamp\":\"2022-08-12T13:30:04.688001387Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"28\",\"timestamp\":\"2022-08-12T13:30:04.688003044Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"29\",\"timestamp\":\"2022-08-12T13:30:04.688014036Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"30\",\"timestamp\":\"2022-08-12T13:30:04.688051272Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"31\",\"timestamp\":\"2022-08-12T13:30:04.688126709Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"32\",\"timestamp\":\"2022-08-12T13:30:04.688128322Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"33\",\"timestamp\":\"2022-08-12T13:30:04.688139583Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"34\",\"timestamp\":\"2022-08-12T13:30:04.68816411Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"35\",\"timestamp\":\"2022-08-12T13:30:04.688234038Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"36\",\"timestamp\":\"2022-08-12T13:30:04.688235549Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"37\",\"timestamp\":\"2022-08-12T13:30:04.68824796Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"38\",\"timestamp\":\"2022-08-12T13:30:04.688278753Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"39\",\"timestamp\":\"2022-08-12T13:30:04.688332263Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"40\",\"timestamp\":\"2022-08-12T13:30:04.688333783Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"41\",\"timestamp\":\"2022-08-12T13:30:04.688349331Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"42\",\"timestamp\":\"2022-08-12T13:30:04.688377769Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"43\",\"timestamp\":\"2022-08-12T13:30:04.688437161Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"44\",\"timestamp\":\"2022-08-12T13:30:04.688439485Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"45\",\"timestamp\":\"2022-08-12T13:30:04.688456515Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"46\",\"timestamp\":\"2022-08-12T13:30:04.68848179Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"47\",\"timestamp\":\"2022-08-12T13:30:04.688559693Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"48\",\"timestamp\":\"2022-08-12T13:30:04.688561281Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"53\",\"timestamp\":\"2022-08-12T13:30:04.688689148Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"54\",\"timestamp\":\"2022-08-12T13:30:04.688714557Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"55\",\"timestamp\":\"2022-08-12T13:30:04.688860562Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"49\",\"timestamp\":\"2022-08-12T13:30:04.688573177Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"56\",\"timestamp\":\"2022-08-12T13:30:04.68886246Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"50\",\"timestamp\":\"2022-08-12T13:30:04.688606432Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"51\",\"timestamp\":\"2022-08-12T13:30:04.688673085Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"52\",\"timestamp\":\"2022-08-12T13:30:04.688675724Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"57\",\"timestamp\":\"2022-08-12T13:30:04.688875227Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"58\",\"timestamp\":\"2022-08-12T13:30:04.688930281Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"59\",\"timestamp\":\"2022-08-12T13:30:04.688975956Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"60\",\"timestamp\":\"2022-08-12T13:30:04.688986533Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"61\",\"timestamp\":\"2022-08-12T13:30:04.688994591Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"62\",\"timestamp\":\"2022-08-12T13:30:04.689021591Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"63\",\"timestamp\":\"2022-08-12T13:30:04.689171143Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"64\",\"timestamp\":\"2022-08-12T13:30:04.689173043Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"65\",\"timestamp\":\"2022-08-12T13:30:04.68918868Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"66\",\"timestamp\":\"2022-08-12T13:30:04.689231633Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"67\",\"timestamp\":\"2022-08-12T13:30:04.689332791Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"68\",\"timestamp\":\"2022-08-12T13:30:04.689334352Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"69\",\"timestamp\":\"2022-08-12T13:30:04.689345198Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"70\",\"timestamp\":\"2022-08-12T13:30:04.68937079Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"71\",\"timestamp\":\"2022-08-12T13:30:04.689463921Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"77\",\"timestamp\":\"2022-08-12T13:30:04.689602894Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"72\",\"timestamp\":\"2022-08-12T13:30:04.689466356Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"78\",\"timestamp\":\"2022-08-12T13:30:04.68963501Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"79\",\"timestamp\":\"2022-08-12T13:30:04.689700114Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"80\",\"timestamp\":\"2022-08-12T13:30:04.689702805Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"73\",\"timestamp\":\"2022-08-12T13:30:04.689484797Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"74\",\"timestamp\":\"2022-08-12T13:30:04.689509529Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"75\",\"timestamp\":\"2022-08-12T13:30:04.689590003Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"76\",\"timestamp\":\"2022-08-12T13:30:04.689591492Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"85\",\"timestamp\":\"2022-08-12T13:30:04.689884469Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"81\",\"timestamp\":\"2022-08-12T13:30:04.689718471Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"86\",\"timestamp\":\"2022-08-12T13:30:04.689930036Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"82\",\"timestamp\":\"2022-08-12T13:30:04.689755218Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"87\",\"timestamp\":\"2022-08-12T13:30:04.690078096Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"83\",\"timestamp\":\"2022-08-12T13:30:04.689868169Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"89\",\"timestamp\":\"2022-08-12T13:30:04.690121194Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"88\",\"timestamp\":\"2022-08-12T13:30:04.690081266Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"84\",\"timestamp\":\"2022-08-12T13:30:04.689870538Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"90\",\"timestamp\":\"2022-08-12T13:30:04.690123522Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"93\",\"timestamp\":\"2022-08-12T13:30:04.690276406Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"91\",\"timestamp\":\"2022-08-12T13:30:04.690263137Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"92\",\"timestamp\":\"2022-08-12T13:30:04.690264843Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"97\",\"timestamp\":\"2022-08-12T13:30:04.690407214Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"101\",\"timestamp\":\"2022-08-12T13:30:04.690536078Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"98\",\"timestamp\":\"2022-08-12T13:30:04.690433197Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"102\",\"timestamp\":\"2022-08-12T13:30:04.690560435Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"99\",\"timestamp\":\"2022-08-12T13:30:04.690522808Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"103\",\"timestamp\":\"2022-08-12T13:30:04.690633135Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"100\",\"timestamp\":\"2022-08-12T13:30:04.690524308Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"104\",\"timestamp\":\"2022-08-12T13:30:04.690634614Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"94\",\"timestamp\":\"2022-08-12T13:30:04.690304932Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"95\",\"timestamp\":\"2022-08-12T13:30:04.690392724Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"96\",\"timestamp\":\"2022-08-12T13:30:04.690394266Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"105\",\"timestamp\":\"2022-08-12T13:30:04.69064582Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"106\",\"timestamp\":\"2022-08-12T13:30:04.690669619Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"107\",\"timestamp\":\"2022-08-12T13:30:04.690753237Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"108\",\"timestamp\":\"2022-08-12T13:30:04.690755122Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"109\",\"timestamp\":\"2022-08-12T13:30:04.690772753Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"110\",\"timestamp\":\"2022-08-12T13:30:04.69080219Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"111\",\"timestamp\":\"2022-08-12T13:30:04.690877097Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"112\",\"timestamp\":\"2022-08-12T13:30:04.690878543Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"113\",\"timestamp\":\"2022-08-12T13:30:04.690893601Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"114\",\"timestamp\":\"2022-08-12T13:30:04.690919509Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"117\",\"timestamp\":\"2022-08-12T13:30:04.691041068Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"118\",\"timestamp\":\"2022-08-12T13:30:04.691063975Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"119\",\"timestamp\":\"2022-08-12T13:30:04.691140054Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"129\",\"timestamp\":\"2022-08-12T13:30:04.691387414Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"120\",\"timestamp\":\"2022-08-12T13:30:04.691141565Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"130\",\"timestamp\":\"2022-08-12T13:30:04.691410889Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"131\",\"timestamp\":\"2022-08-12T13:30:04.691497238Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"132\",\"timestamp\":\"2022-08-12T13:30:04.691498657Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"125\",\"timestamp\":\"2022-08-12T13:30:04.691282187Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"126\",\"timestamp\":\"2022-08-12T13:30:04.691305594Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"121\",\"timestamp\":\"2022-08-12T13:30:04.691153235Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"127\",\"timestamp\":\"2022-08-12T13:30:04.691372941Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"122\",\"timestamp\":\"2022-08-12T13:30:04.691176001Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"128\",\"timestamp\":\"2022-08-12T13:30:04.69137442Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"123\",\"timestamp\":\"2022-08-12T13:30:04.691270038Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"124\",\"timestamp\":\"2022-08-12T13:30:04.691271381Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"115\",\"timestamp\":\"2022-08-12T13:30:04.691026241Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"116\",\"timestamp\":\"2022-08-12T13:30:04.691028254Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"133\",\"timestamp\":\"2022-08-12T13:30:04.691509865Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"134\",\"timestamp\":\"2022-08-12T13:30:04.691531912Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"135\",\"timestamp\":\"2022-08-12T13:30:04.691603332Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"136\",\"timestamp\":\"2022-08-12T13:30:04.691604736Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"137\",\"timestamp\":\"2022-08-12T13:30:04.691618847Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"138\",\"timestamp\":\"2022-08-12T13:30:04.691642663Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"139\",\"timestamp\":\"2022-08-12T13:30:04.691731628Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"140\",\"timestamp\":\"2022-08-12T13:30:04.691732919Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"141\",\"timestamp\":\"2022-08-12T13:30:04.691743131Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"142\",\"timestamp\":\"2022-08-12T13:30:04.691765184Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"143\",\"timestamp\":\"2022-08-12T13:30:04.691832098Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"144\",\"timestamp\":\"2022-08-12T13:30:04.691834198Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"145\",\"timestamp\":\"2022-08-12T13:30:04.691845287Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"146\",\"timestamp\":\"2022-08-12T13:30:04.69187195Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"147\",\"timestamp\":\"2022-08-12T13:30:04.691956577Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"148\",\"timestamp\":\"2022-08-12T13:30:04.691958009Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"149\",\"timestamp\":\"2022-08-12T13:30:04.691969147Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"150\",\"timestamp\":\"2022-08-12T13:30:04.69199273Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"151\",\"timestamp\":\"2022-08-12T13:30:04.692062796Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"152\",\"timestamp\":\"2022-08-12T13:30:04.692064213Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"153\",\"timestamp\":\"2022-08-12T13:30:04.692076476Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"157\",\"timestamp\":\"2022-08-12T13:30:04.692195395Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"154\",\"timestamp\":\"2022-08-12T13:30:04.692100205Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"158\",\"timestamp\":\"2022-08-12T13:30:04.692241973Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"155\",\"timestamp\":\"2022-08-12T13:30:04.692178771Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"159\",\"timestamp\":\"2022-08-12T13:30:04.692298151Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"156\",\"timestamp\":\"2022-08-12T13:30:04.692180225Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"160\",\"timestamp\":\"2022-08-12T13:30:04.692301626Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"161\",\"timestamp\":\"2022-08-12T13:30:04.692317326Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"176\",\"timestamp\":\"2022-08-12T13:30:04.692875814Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"162\",\"timestamp\":\"2022-08-12T13:30:04.692345913Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"172\",\"timestamp\":\"2022-08-12T13:30:04.692728279Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"163\",\"timestamp\":\"2022-08-12T13:30:04.69242628Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"173\",\"timestamp\":\"2022-08-12T13:30:04.692741323Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"164\",\"timestamp\":\"2022-08-12T13:30:04.692427856Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"174\",\"timestamp\":\"2022-08-12T13:30:04.692862163Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"175\",\"timestamp\":\"2022-08-12T13:30:04.692863706Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"169\",\"timestamp\":\"2022-08-12T13:30:04.69266267Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"170\",\"timestamp\":\"2022-08-12T13:30:04.692677926Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"171\",\"timestamp\":\"2022-08-12T13:30:04.692726759Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"165\",\"timestamp\":\"2022-08-12T13:30:04.69243928Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"166\",\"timestamp\":\"2022-08-12T13:30:04.692466086Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"167\",\"timestamp\":\"2022-08-12T13:30:04.692537679Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"177\",\"timestamp\":\"2022-08-12T13:30:04.692907917Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"168\",\"timestamp\":\"2022-08-12T13:30:04.692572346Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"178\",\"timestamp\":\"2022-08-12T13:30:04.692983835Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"179\",\"timestamp\":\"2022-08-12T13:30:04.692985498Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"180\",\"timestamp\":\"2022-08-12T13:30:04.69299595Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"181\",\"timestamp\":\"2022-08-12T13:30:04.693022057Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"182\",\"timestamp\":\"2022-08-12T13:30:04.693098237Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"184\",\"timestamp\":\"2022-08-12T13:30:04.693111014Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"183\",\"timestamp\":\"2022-08-12T13:30:04.693100161Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"192\",\"timestamp\":\"2022-08-12T13:30:04.693415579Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"193\",\"timestamp\":\"2022-08-12T13:30:04.693417373Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"196\",\"timestamp\":\"2022-08-12T13:30:04.693524935Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"194\",\"timestamp\":\"2022-08-12T13:30:04.693510485Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"197\",\"timestamp\":\"2022-08-12T13:30:04.693553832Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"195\",\"timestamp\":\"2022-08-12T13:30:04.693512123Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"198\",\"timestamp\":\"2022-08-12T13:30:04.693635481Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"199\",\"timestamp\":\"2022-08-12T13:30:04.693637098Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"188\",\"timestamp\":\"2022-08-12T13:30:04.693243745Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"189\",\"timestamp\":\"2022-08-12T13:30:04.69328921Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"190\",\"timestamp\":\"2022-08-12T13:30:04.693356023Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"185\",\"timestamp\":\"2022-08-12T13:30:04.693141543Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"191\",\"timestamp\":\"2022-08-12T13:30:04.693357893Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"186\",\"timestamp\":\"2022-08-12T13:30:04.693224619Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"200\",\"timestamp\":\"2022-08-12T13:30:04.693648973Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"187\",\"timestamp\":\"2022-08-12T13:30:04.693226683Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"201\",\"timestamp\":\"2022-08-12T13:30:04.693679566Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"202\",\"timestamp\":\"2022-08-12T13:30:04.693749798Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"203\",\"timestamp\":\"2022-08-12T13:30:04.69375125Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"204\",\"timestamp\":\"2022-08-12T13:30:04.693762378Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"205\",\"timestamp\":\"2022-08-12T13:30:04.693787008Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"206\",\"timestamp\":\"2022-08-12T13:30:04.693847183Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"208\",\"timestamp\":\"2022-08-12T13:30:04.693859593Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"209\",\"timestamp\":\"2022-08-12T13:30:04.693885136Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"216\",\"timestamp\":\"2022-08-12T13:30:04.694060214Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"210\",\"timestamp\":\"2022-08-12T13:30:04.693945469Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"217\",\"timestamp\":\"2022-08-12T13:30:04.694083789Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"211\",\"timestamp\":\"2022-08-12T13:30:04.693947282Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"218\",\"timestamp\":\"2022-08-12T13:30:04.694145516Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"219\",\"timestamp\":\"2022-08-12T13:30:04.694146949Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"212\",\"timestamp\":\"2022-08-12T13:30:04.693963187Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"213\",\"timestamp\":\"2022-08-12T13:30:04.693987556Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"214\",\"timestamp\":\"2022-08-12T13:30:04.694046409Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"207\",\"timestamp\":\"2022-08-12T13:30:04.693848965Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"215\",\"timestamp\":\"2022-08-12T13:30:04.694047924Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"220\",\"timestamp\":\"2022-08-12T13:30:04.694158589Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"221\",\"timestamp\":\"2022-08-12T13:30:04.69418191Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"222\",\"timestamp\":\"2022-08-12T13:30:04.694253672Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"223\",\"timestamp\":\"2022-08-12T13:30:04.694255467Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"224\",\"timestamp\":\"2022-08-12T13:30:04.69426637Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"225\",\"timestamp\":\"2022-08-12T13:30:04.694292993Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"226\",\"timestamp\":\"2022-08-12T13:30:04.694349649Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"228\",\"timestamp\":\"2022-08-12T13:30:04.694361903Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"227\",\"timestamp\":\"2022-08-12T13:30:04.694351011Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"229\",\"timestamp\":\"2022-08-12T13:30:04.694384807Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"230\",\"timestamp\":\"2022-08-12T13:30:04.694447415Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"231\",\"timestamp\":\"2022-08-12T13:30:04.69444895Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"240\",\"timestamp\":\"2022-08-12T13:30:04.69465744Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"241\",\"timestamp\":\"2022-08-12T13:30:04.694683152Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"236\",\"timestamp\":\"2022-08-12T13:30:04.694558979Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"242\",\"timestamp\":\"2022-08-12T13:30:04.694743634Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"237\",\"timestamp\":\"2022-08-12T13:30:04.69458339Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"243\",\"timestamp\":\"2022-08-12T13:30:04.694744993Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"238\",\"timestamp\":\"2022-08-12T13:30:04.694644799Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"239\",\"timestamp\":\"2022-08-12T13:30:04.694646165Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"232\",\"timestamp\":\"2022-08-12T13:30:04.694460741Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"233\",\"timestamp\":\"2022-08-12T13:30:04.694484631Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"234\",\"timestamp\":\"2022-08-12T13:30:04.6945459Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"235\",\"timestamp\":\"2022-08-12T13:30:04.694547231Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"244\",\"timestamp\":\"2022-08-12T13:30:04.694757562Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"245\",\"timestamp\":\"2022-08-12T13:30:04.694781236Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"246\",\"timestamp\":\"2022-08-12T13:30:04.694842613Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"247\",\"timestamp\":\"2022-08-12T13:30:04.694844304Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"248\",\"timestamp\":\"2022-08-12T13:30:04.694855437Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"249\",\"timestamp\":\"2022-08-12T13:30:04.694879208Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"250\",\"timestamp\":\"2022-08-12T13:30:04.695113263Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"251\",\"timestamp\":\"2022-08-12T13:30:04.695114808Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"252\",\"timestamp\":\"2022-08-12T13:30:04.695126598Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"253\",\"timestamp\":\"2022-08-12T13:30:04.695153021Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"254\",\"timestamp\":\"2022-08-12T13:30:04.695215823Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"260\",\"timestamp\":\"2022-08-12T13:30:04.695334059Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"255\",\"timestamp\":\"2022-08-12T13:30:04.695217257Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"261\",\"timestamp\":\"2022-08-12T13:30:04.69535803Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"262\",\"timestamp\":\"2022-08-12T13:30:04.695419545Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"256\",\"timestamp\":\"2022-08-12T13:30:04.695228423Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"263\",\"timestamp\":\"2022-08-12T13:30:04.695420919Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"257\",\"timestamp\":\"2022-08-12T13:30:04.695252559Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"258\",\"timestamp\":\"2022-08-12T13:30:04.695322097Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"259\",\"timestamp\":\"2022-08-12T13:30:04.695323489Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"264\",\"timestamp\":\"2022-08-12T13:30:04.695432895Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"265\",\"timestamp\":\"2022-08-12T13:30:04.69545634Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"266\",\"timestamp\":\"2022-08-12T13:30:04.695518181Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"268\",\"timestamp\":\"2022-08-12T13:30:04.695531492Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"267\",\"timestamp\":\"2022-08-12T13:30:04.69551958Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"269\",\"timestamp\":\"2022-08-12T13:30:04.695555392Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"270\",\"timestamp\":\"2022-08-12T13:30:04.695618372Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"271\",\"timestamp\":\"2022-08-12T13:30:04.695619772Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"272\",\"timestamp\":\"2022-08-12T13:30:04.695630522Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"273\",\"timestamp\":\"2022-08-12T13:30:04.695655147Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"274\",\"timestamp\":\"2022-08-12T13:30:04.695714559Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"275\",\"timestamp\":\"2022-08-12T13:30:04.695715969Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"280\",\"timestamp\":\"2022-08-12T13:30:04.695825707Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"281\",\"timestamp\":\"2022-08-12T13:30:04.695849009Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"282\",\"timestamp\":\"2022-08-12T13:30:04.695911493Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"276\",\"timestamp\":\"2022-08-12T13:30:04.695728339Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"283\",\"timestamp\":\"2022-08-12T13:30:04.695912958Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"277\",\"timestamp\":\"2022-08-12T13:30:04.695751608Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"278\",\"timestamp\":\"2022-08-12T13:30:04.695814162Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"279\",\"timestamp\":\"2022-08-12T13:30:04.69581555Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"284\",\"timestamp\":\"2022-08-12T13:30:04.695924267Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"285\",\"timestamp\":\"2022-08-12T13:30:04.695947645Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"286\",\"timestamp\":\"2022-08-12T13:30:04.696008189Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"287\",\"timestamp\":\"2022-08-12T13:30:04.696009573Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"288\",\"timestamp\":\"2022-08-12T13:30:04.696020269Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"289\",\"timestamp\":\"2022-08-12T13:30:04.69604242Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"290\",\"timestamp\":\"2022-08-12T13:30:04.696093545Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"291\",\"timestamp\":\"2022-08-12T13:30:04.696095061Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"292\",\"timestamp\":\"2022-08-12T13:30:04.696105202Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"293\",\"timestamp\":\"2022-08-12T13:30:04.69612826Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"294\",\"timestamp\":\"2022-08-12T13:30:04.696195685Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"296\",\"timestamp\":\"2022-08-12T13:30:04.696227173Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"295\",\"timestamp\":\"2022-08-12T13:30:04.696197817Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"297\",\"timestamp\":\"2022-08-12T13:30:04.696242784Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"298\",\"timestamp\":\"2022-08-12T13:30:04.69638203Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"299\",\"timestamp\":\"2022-08-12T13:30:04.69638378Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"300\",\"timestamp\":\"2022-08-12T13:30:04.696398687Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"301\",\"timestamp\":\"2022-08-12T13:30:04.696430352Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"302\",\"timestamp\":\"2022-08-12T13:30:04.696518311Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"303\",\"timestamp\":\"2022-08-12T13:30:04.69652062Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"304\",\"timestamp\":\"2022-08-12T13:30:04.696540129Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"305\",\"timestamp\":\"2022-08-12T13:30:04.696590149Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"306\",\"timestamp\":\"2022-08-12T13:30:04.696660475Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"307\",\"timestamp\":\"2022-08-12T13:30:04.696662012Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"308\",\"timestamp\":\"2022-08-12T13:30:04.696690796Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"309\",\"timestamp\":\"2022-08-12T13:30:04.696730877Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"310\",\"timestamp\":\"2022-08-12T13:30:04.696830771Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"311\",\"timestamp\":\"2022-08-12T13:30:04.696832305Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"312\",\"timestamp\":\"2022-08-12T13:30:04.696845426Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"313\",\"timestamp\":\"2022-08-12T13:30:04.696873105Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"314\",\"timestamp\":\"2022-08-12T13:30:04.69695739Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"315\",\"timestamp\":\"2022-08-12T13:30:04.696958961Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"316\",\"timestamp\":\"2022-08-12T13:30:04.696972666Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"317\",\"timestamp\":\"2022-08-12T13:30:04.697001155Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"318\",\"timestamp\":\"2022-08-12T13:30:04.697072945Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"319\",\"timestamp\":\"2022-08-12T13:30:04.697074408Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"320\",\"timestamp\":\"2022-08-12T13:30:04.697085567Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"321\",\"timestamp\":\"2022-08-12T13:30:04.697108718Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"322\",\"timestamp\":\"2022-08-12T13:30:04.697166986Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"323\",\"timestamp\":\"2022-08-12T13:30:04.697168337Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"324\",\"timestamp\":\"2022-08-12T13:30:04.697180091Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"325\",\"timestamp\":\"2022-08-12T13:30:04.697202922Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"326\",\"timestamp\":\"2022-08-12T13:30:04.697268647Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"327\",\"timestamp\":\"2022-08-12T13:30:04.697269994Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"328\",\"timestamp\":\"2022-08-12T13:30:04.697281008Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"329\",\"timestamp\":\"2022-08-12T13:30:04.697304114Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"330\",\"timestamp\":\"2022-08-12T13:30:04.69735311Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"332\",\"timestamp\":\"2022-08-12T13:30:04.697364732Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"331\",\"timestamp\":\"2022-08-12T13:30:04.697354421Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"333\",\"timestamp\":\"2022-08-12T13:30:04.697387362Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"334\",\"timestamp\":\"2022-08-12T13:30:04.697464404Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"335\",\"timestamp\":\"2022-08-12T13:30:04.69746577Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"336\",\"timestamp\":\"2022-08-12T13:30:04.697476992Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"337\",\"timestamp\":\"2022-08-12T13:30:04.697499001Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"338\",\"timestamp\":\"2022-08-12T13:30:04.697548663Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"339\",\"timestamp\":\"2022-08-12T13:30:04.697550048Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"340\",\"timestamp\":\"2022-08-12T13:30:04.697560534Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"341\",\"timestamp\":\"2022-08-12T13:30:04.69758234Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"342\",\"timestamp\":\"2022-08-12T13:30:04.697637478Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"343\",\"timestamp\":\"2022-08-12T13:30:04.697638783Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"344\",\"timestamp\":\"2022-08-12T13:30:04.697652488Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"345\",\"timestamp\":\"2022-08-12T13:30:04.69767455Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"346\",\"timestamp\":\"2022-08-12T13:30:04.697733548Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"347\",\"timestamp\":\"2022-08-12T13:30:04.697734904Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"348\",\"timestamp\":\"2022-08-12T13:30:04.697749361Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"349\",\"timestamp\":\"2022-08-12T13:30:04.697774665Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"350\",\"timestamp\":\"2022-08-12T13:30:04.697829909Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"351\",\"timestamp\":\"2022-08-12T13:30:04.697831278Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"352\",\"timestamp\":\"2022-08-12T13:30:04.697843048Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"353\",\"timestamp\":\"2022-08-12T13:30:04.697865318Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"354\",\"timestamp\":\"2022-08-12T13:30:04.697914563Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"355\",\"timestamp\":\"2022-08-12T13:30:04.697915969Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"356\",\"timestamp\":\"2022-08-12T13:30:04.697926155Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"357\",\"timestamp\":\"2022-08-12T13:30:04.697949141Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"358\",\"timestamp\":\"2022-08-12T13:30:04.697999614Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"359\",\"timestamp\":\"2022-08-12T13:30:04.698000967Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"360\",\"timestamp\":\"2022-08-12T13:30:04.698010865Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"361\",\"timestamp\":\"2022-08-12T13:30:04.698033605Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"362\",\"timestamp\":\"2022-08-12T13:30:04.698083348Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"363\",\"timestamp\":\"2022-08-12T13:30:04.698084693Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"364\",\"timestamp\":\"2022-08-12T13:30:04.698095857Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"365\",\"timestamp\":\"2022-08-12T13:30:04.698117936Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"366\",\"timestamp\":\"2022-08-12T13:30:04.698168574Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"367\",\"timestamp\":\"2022-08-12T13:30:04.698169972Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"368\",\"timestamp\":\"2022-08-12T13:30:04.698180376Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"369\",\"timestamp\":\"2022-08-12T13:30:04.698201946Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"370\",\"timestamp\":\"2022-08-12T13:30:04.698254245Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"371\",\"timestamp\":\"2022-08-12T13:30:04.698255635Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"372\",\"timestamp\":\"2022-08-12T13:30:04.698266316Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"373\",\"timestamp\":\"2022-08-12T13:30:04.698287594Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"374\",\"timestamp\":\"2022-08-12T13:30:04.698339099Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"375\",\"timestamp\":\"2022-08-12T13:30:04.698340447Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"376\",\"timestamp\":\"2022-08-12T13:30:04.698352046Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"377\",\"timestamp\":\"2022-08-12T13:30:04.698373786Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"378\",\"timestamp\":\"2022-08-12T13:30:04.698425574Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"379\",\"timestamp\":\"2022-08-12T13:30:04.698426889Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"380\",\"timestamp\":\"2022-08-12T13:30:04.698438744Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"381\",\"timestamp\":\"2022-08-12T13:30:04.698461298Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"382\",\"timestamp\":\"2022-08-12T13:30:04.698527063Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"383\",\"timestamp\":\"2022-08-12T13:30:04.698528604Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"384\",\"timestamp\":\"2022-08-12T13:30:04.698539973Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"385\",\"timestamp\":\"2022-08-12T13:30:04.698562009Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"386\",\"timestamp\":\"2022-08-12T13:30:04.698610715Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"387\",\"timestamp\":\"2022-08-12T13:30:04.698612126Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"388\",\"timestamp\":\"2022-08-12T13:30:04.698623135Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"389\",\"timestamp\":\"2022-08-12T13:30:04.698645389Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"390\",\"timestamp\":\"2022-08-12T13:30:04.698697102Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"391\",\"timestamp\":\"2022-08-12T13:30:04.698698497Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"392\",\"timestamp\":\"2022-08-12T13:30:04.698709277Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"393\",\"timestamp\":\"2022-08-12T13:30:04.698731161Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"394\",\"timestamp\":\"2022-08-12T13:30:04.698783297Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"395\",\"timestamp\":\"2022-08-12T13:30:04.698784691Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"396\",\"timestamp\":\"2022-08-12T13:30:04.698795352Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"397\",\"timestamp\":\"2022-08-12T13:30:04.698817557Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"398\",\"timestamp\":\"2022-08-12T13:30:04.698869718Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"399\",\"timestamp\":\"2022-08-12T13:30:04.698871062Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"400\",\"timestamp\":\"2022-08-12T13:30:04.698881666Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"401\",\"timestamp\":\"2022-08-12T13:30:04.698902766Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"402\",\"timestamp\":\"2022-08-12T13:30:04.698954504Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"403\",\"timestamp\":\"2022-08-12T13:30:04.698955895Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"404\",\"timestamp\":\"2022-08-12T13:30:04.698966515Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"405\",\"timestamp\":\"2022-08-12T13:30:04.698988908Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"406\",\"timestamp\":\"2022-08-12T13:30:04.699042398Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"407\",\"timestamp\":\"2022-08-12T13:30:04.699043801Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"408\",\"timestamp\":\"2022-08-12T13:30:04.699055661Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"409\",\"timestamp\":\"2022-08-12T13:30:04.699107001Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"410\",\"timestamp\":\"2022-08-12T13:30:04.699186359Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"411\",\"timestamp\":\"2022-08-12T13:30:04.699197581Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"412\",\"timestamp\":\"2022-08-12T13:30:04.699205971Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"413\",\"timestamp\":\"2022-08-12T13:30:04.699217868Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"414\",\"timestamp\":\"2022-08-12T13:30:04.699302302Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"415\",\"timestamp\":\"2022-08-12T13:30:04.699304503Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"416\",\"timestamp\":\"2022-08-12T13:30:04.699317968Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"417\",\"timestamp\":\"2022-08-12T13:30:04.699346817Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"418\",\"timestamp\":\"2022-08-12T13:30:04.699428241Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"419\",\"timestamp\":\"2022-08-12T13:30:04.69942974Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"420\",\"timestamp\":\"2022-08-12T13:30:04.699441257Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"421\",\"timestamp\":\"2022-08-12T13:30:04.699464746Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"422\",\"timestamp\":\"2022-08-12T13:30:04.699548025Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"423\",\"timestamp\":\"2022-08-12T13:30:04.699549752Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"424\",\"timestamp\":\"2022-08-12T13:30:04.699560608Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"425\",\"timestamp\":\"2022-08-12T13:30:04.699584562Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"426\",\"timestamp\":\"2022-08-12T13:30:04.699672833Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"427\",\"timestamp\":\"2022-08-12T13:30:04.699674218Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"428\",\"timestamp\":\"2022-08-12T13:30:04.699686007Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"429\",\"timestamp\":\"2022-08-12T13:30:04.699709147Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"430\",\"timestamp\":\"2022-08-12T13:30:04.699790251Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"431\",\"timestamp\":\"2022-08-12T13:30:04.69979235Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"432\",\"timestamp\":\"2022-08-12T13:30:04.699809916Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"433\",\"timestamp\":\"2022-08-12T13:30:04.699856973Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"434\",\"timestamp\":\"2022-08-12T13:30:04.699917961Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"435\",\"timestamp\":\"2022-08-12T13:30:04.699925398Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"436\",\"timestamp\":\"2022-08-12T13:30:04.699959003Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"437\",\"timestamp\":\"2022-08-12T13:30:04.699960449Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"438\",\"timestamp\":\"2022-08-12T13:30:04.700031357Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"439\",\"timestamp\":\"2022-08-12T13:30:04.700032901Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"440\",\"timestamp\":\"2022-08-12T13:30:04.700045687Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"441\",\"timestamp\":\"2022-08-12T13:30:04.700070031Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"442\",\"timestamp\":\"2022-08-12T13:30:04.700146267Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"443\",\"timestamp\":\"2022-08-12T13:30:04.700147735Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"444\",\"timestamp\":\"2022-08-12T13:30:04.7001586Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"445\",\"timestamp\":\"2022-08-12T13:30:04.70018244Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"446\",\"timestamp\":\"2022-08-12T13:30:04.700243742Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"447\",\"timestamp\":\"2022-08-12T13:30:04.700245134Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"448\",\"timestamp\":\"2022-08-12T13:30:04.700257598Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"449\",\"timestamp\":\"2022-08-12T13:30:04.700280541Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"450\",\"timestamp\":\"2022-08-12T13:30:04.700343448Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"451\",\"timestamp\":\"2022-08-12T13:30:04.700345049Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"452\",\"timestamp\":\"2022-08-12T13:30:04.700357818Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"453\",\"timestamp\":\"2022-08-12T13:30:04.700382802Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"454\",\"timestamp\":\"2022-08-12T13:30:04.700450589Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"455\",\"timestamp\":\"2022-08-12T13:30:04.700452059Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"456\",\"timestamp\":\"2022-08-12T13:30:04.700466067Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"457\",\"timestamp\":\"2022-08-12T13:30:04.700490112Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"458\",\"timestamp\":\"2022-08-12T13:30:04.700553243Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"459\",\"timestamp\":\"2022-08-12T13:30:04.700554651Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"460\",\"timestamp\":\"2022-08-12T13:30:04.700567196Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"461\",\"timestamp\":\"2022-08-12T13:30:04.70059108Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"462\",\"timestamp\":\"2022-08-12T13:30:04.70067286Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"463\",\"timestamp\":\"2022-08-12T13:30:04.700674334Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"464\",\"timestamp\":\"2022-08-12T13:30:04.700685415Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"465\",\"timestamp\":\"2022-08-12T13:30:04.700710005Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"466\",\"timestamp\":\"2022-08-12T13:30:04.700767726Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"467\",\"timestamp\":\"2022-08-12T13:30:04.70076945Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"468\",\"timestamp\":\"2022-08-12T13:30:04.700804243Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"469\",\"timestamp\":\"2022-08-12T13:30:04.700827468Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"470\",\"timestamp\":\"2022-08-12T13:30:04.700894527Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"471\",\"timestamp\":\"2022-08-12T13:30:04.700896113Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"472\",\"timestamp\":\"2022-08-12T13:30:04.700907008Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"473\",\"timestamp\":\"2022-08-12T13:30:04.700931202Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"474\",\"timestamp\":\"2022-08-12T13:30:04.7010021Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"475\",\"timestamp\":\"2022-08-12T13:30:04.701003644Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"476\",\"timestamp\":\"2022-08-12T13:30:04.701014805Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"477\",\"timestamp\":\"2022-08-12T13:30:04.701039215Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"478\",\"timestamp\":\"2022-08-12T13:30:04.70110792Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"479\",\"timestamp\":\"2022-08-12T13:30:04.701109477Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"480\",\"timestamp\":\"2022-08-12T13:30:04.701121995Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"481\",\"timestamp\":\"2022-08-12T13:30:04.701146371Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"482\",\"timestamp\":\"2022-08-12T13:30:04.701201514Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"483\",\"timestamp\":\"2022-08-12T13:30:04.701202942Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"484\",\"timestamp\":\"2022-08-12T13:30:04.701213855Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"485\",\"timestamp\":\"2022-08-12T13:30:04.701238883Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"486\",\"timestamp\":\"2022-08-12T13:30:04.701298431Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"487\",\"timestamp\":\"2022-08-12T13:30:04.701299928Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"488\",\"timestamp\":\"2022-08-12T13:30:04.701310874Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"489\",\"timestamp\":\"2022-08-12T13:30:04.701335722Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"490\",\"timestamp\":\"2022-08-12T13:30:04.701381974Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"491\",\"timestamp\":\"2022-08-12T13:30:04.70138328Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"492\",\"timestamp\":\"2022-08-12T13:30:04.701395934Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"493\",\"timestamp\":\"2022-08-12T13:30:04.701419614Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"494\",\"timestamp\":\"2022-08-12T13:30:04.701467277Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"495\",\"timestamp\":\"2022-08-12T13:30:04.701468843Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"496\",\"timestamp\":\"2022-08-12T13:30:04.701484172Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"497\",\"timestamp\":\"2022-08-12T13:30:04.70150655Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"498\",\"timestamp\":\"2022-08-12T13:30:04.701561836Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"499\",\"timestamp\":\"2022-08-12T13:30:04.701563363Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"500\",\"timestamp\":\"2022-08-12T13:30:04.70157438Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"501\",\"timestamp\":\"2022-08-12T13:30:04.70159676Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"502\",\"timestamp\":\"2022-08-12T13:30:04.701645134Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"503\",\"timestamp\":\"2022-08-12T13:30:04.701646504Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"504\",\"timestamp\":\"2022-08-12T13:30:04.701657604Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"505\",\"timestamp\":\"2022-08-12T13:30:04.701679223Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"506\",\"timestamp\":\"2022-08-12T13:30:04.701746374Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"507\",\"timestamp\":\"2022-08-12T13:30:04.701747996Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"508\",\"timestamp\":\"2022-08-12T13:30:04.701771132Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"509\",\"timestamp\":\"2022-08-12T13:30:04.701793976Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"510\",\"timestamp\":\"2022-08-12T13:30:04.701857158Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"511\",\"timestamp\":\"2022-08-12T13:30:04.701858648Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"512\",\"timestamp\":\"2022-08-12T13:30:04.701869489Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"513\",\"timestamp\":\"2022-08-12T13:30:04.70189186Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"514\",\"timestamp\":\"2022-08-12T13:30:04.70194361Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"515\",\"timestamp\":\"2022-08-12T13:30:04.701944928Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"516\",\"timestamp\":\"2022-08-12T13:30:04.701956241Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"517\",\"timestamp\":\"2022-08-12T13:30:04.7019784Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"518\",\"timestamp\":\"2022-08-12T13:30:04.702037463Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"519\",\"timestamp\":\"2022-08-12T13:30:04.70203889Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"520\",\"timestamp\":\"2022-08-12T13:30:04.702051432Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"521\",\"timestamp\":\"2022-08-12T13:30:04.702074461Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"522\",\"timestamp\":\"2022-08-12T13:30:04.702129706Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"523\",\"timestamp\":\"2022-08-12T13:30:04.702131224Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"524\",\"timestamp\":\"2022-08-12T13:30:04.702159656Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"525\",\"timestamp\":\"2022-08-12T13:30:04.702161136Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"526\",\"timestamp\":\"2022-08-12T13:30:04.702212905Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"527\",\"timestamp\":\"2022-08-12T13:30:04.702214393Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"528\",\"timestamp\":\"2022-08-12T13:30:04.702225942Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"529\",\"timestamp\":\"2022-08-12T13:30:04.702250332Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"530\",\"timestamp\":\"2022-08-12T13:30:04.70230389Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"531\",\"timestamp\":\"2022-08-12T13:30:04.702305398Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"532\",\"timestamp\":\"2022-08-12T13:30:04.702315851Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"533\",\"timestamp\":\"2022-08-12T13:30:04.702338206Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"534\",\"timestamp\":\"2022-08-12T13:30:04.70239301Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"535\",\"timestamp\":\"2022-08-12T13:30:04.702394533Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"536\",\"timestamp\":\"2022-08-12T13:30:04.702405636Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"537\",\"timestamp\":\"2022-08-12T13:30:04.702427381Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"538\",\"timestamp\":\"2022-08-12T13:30:04.702481535Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"539\",\"timestamp\":\"2022-08-12T13:30:04.702482826Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"540\",\"timestamp\":\"2022-08-12T13:30:04.702493337Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"541\",\"timestamp\":\"2022-08-12T13:30:04.702515871Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"542\",\"timestamp\":\"2022-08-12T13:30:04.702570853Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"543\",\"timestamp\":\"2022-08-12T13:30:04.70257297Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"544\",\"timestamp\":\"2022-08-12T13:30:04.702584051Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"545\",\"timestamp\":\"2022-08-12T13:30:04.702607414Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"546\",\"timestamp\":\"2022-08-12T13:30:04.702661289Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"547\",\"timestamp\":\"2022-08-12T13:30:04.702662817Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"548\",\"timestamp\":\"2022-08-12T13:30:04.702672982Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"549\",\"timestamp\":\"2022-08-12T13:30:04.702696363Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"550\",\"timestamp\":\"2022-08-12T13:30:04.702745796Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"551\",\"timestamp\":\"2022-08-12T13:30:04.702747264Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"552\",\"timestamp\":\"2022-08-12T13:30:04.70275758Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"553\",\"timestamp\":\"2022-08-12T13:30:04.702780471Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"554\",\"timestamp\":\"2022-08-12T13:30:04.702845903Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"555\",\"timestamp\":\"2022-08-12T13:30:04.70284778Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"556\",\"timestamp\":\"2022-08-12T13:30:04.70286042Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"557\",\"timestamp\":\"2022-08-12T13:30:04.70290041Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"558\",\"timestamp\":\"2022-08-12T13:30:04.702953273Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"559\",\"timestamp\":\"2022-08-12T13:30:04.702955295Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"560\",\"timestamp\":\"2022-08-12T13:30:04.702984203Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"561\",\"timestamp\":\"2022-08-12T13:30:04.703046168Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"562\",\"timestamp\":\"2022-08-12T13:30:04.703094441Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"563\",\"timestamp\":\"2022-08-12T13:30:04.703096131Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"564\",\"timestamp\":\"2022-08-12T13:30:04.703108827Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"565\",\"timestamp\":\"2022-08-12T13:30:04.703151795Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"566\",\"timestamp\":\"2022-08-12T13:30:04.703209725Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"567\",\"timestamp\":\"2022-08-12T13:30:04.703211279Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"568\",\"timestamp\":\"2022-08-12T13:30:04.703225085Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"569\",\"timestamp\":\"2022-08-12T13:30:04.703257313Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"570\",\"timestamp\":\"2022-08-12T13:30:04.703332939Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"571\",\"timestamp\":\"2022-08-12T13:30:04.703334475Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"572\",\"timestamp\":\"2022-08-12T13:30:04.703345816Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"573\",\"timestamp\":\"2022-08-12T13:30:04.703371928Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"574\",\"timestamp\":\"2022-08-12T13:30:04.703438519Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"575\",\"timestamp\":\"2022-08-12T13:30:04.703440095Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"576\",\"timestamp\":\"2022-08-12T13:30:04.703453879Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"577\",\"timestamp\":\"2022-08-12T13:30:04.703482659Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"578\",\"timestamp\":\"2022-08-12T13:30:04.70359583Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"579\",\"timestamp\":\"2022-08-12T13:30:04.703597317Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"580\",\"timestamp\":\"2022-08-12T13:30:04.703609203Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"581\",\"timestamp\":\"2022-08-12T13:30:04.703640093Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"582\",\"timestamp\":\"2022-08-12T13:30:04.703742086Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"583\",\"timestamp\":\"2022-08-12T13:30:04.703743503Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"584\",\"timestamp\":\"2022-08-12T13:30:04.70375489Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"585\",\"timestamp\":\"2022-08-12T13:30:04.703779561Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"586\",\"timestamp\":\"2022-08-12T13:30:04.703890052Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"587\",\"timestamp\":\"2022-08-12T13:30:04.703891779Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"588\",\"timestamp\":\"2022-08-12T13:30:04.703907719Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"589\",\"timestamp\":\"2022-08-12T13:30:04.703953119Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"590\",\"timestamp\":\"2022-08-12T13:30:04.704059216Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"591\",\"timestamp\":\"2022-08-12T13:30:04.704060732Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"592\",\"timestamp\":\"2022-08-12T13:30:04.704072287Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"593\",\"timestamp\":\"2022-08-12T13:30:04.70410801Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"594\",\"timestamp\":\"2022-08-12T13:30:04.704180498Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"595\",\"timestamp\":\"2022-08-12T13:30:04.704182009Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"596\",\"timestamp\":\"2022-08-12T13:30:04.704193724Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"597\",\"timestamp\":\"2022-08-12T13:30:04.704217116Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"598\",\"timestamp\":\"2022-08-12T13:30:04.704293989Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"599\",\"timestamp\":\"2022-08-12T13:30:04.704295347Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"600\",\"timestamp\":\"2022-08-12T13:30:04.704308533Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"601\",\"timestamp\":\"2022-08-12T13:30:04.704332043Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"602\",\"timestamp\":\"2022-08-12T13:30:04.7044051Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"603\",\"timestamp\":\"2022-08-12T13:30:04.704406453Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"604\",\"timestamp\":\"2022-08-12T13:30:04.704417353Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"605\",\"timestamp\":\"2022-08-12T13:30:04.704440176Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"606\",\"timestamp\":\"2022-08-12T13:30:04.704519933Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"607\",\"timestamp\":\"2022-08-12T13:30:04.70452133Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"608\",\"timestamp\":\"2022-08-12T13:30:04.704534809Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"609\",\"timestamp\":\"2022-08-12T13:30:04.704558533Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"610\",\"timestamp\":\"2022-08-12T13:30:04.704624173Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"611\",\"timestamp\":\"2022-08-12T13:30:04.704625573Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"612\",\"timestamp\":\"2022-08-12T13:30:04.704636102Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"613\",\"timestamp\":\"2022-08-12T13:30:04.704657866Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"614\",\"timestamp\":\"2022-08-12T13:30:04.704714659Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"615\",\"timestamp\":\"2022-08-12T13:30:04.70471608Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"616\",\"timestamp\":\"2022-08-12T13:30:04.704727384Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"617\",\"timestamp\":\"2022-08-12T13:30:04.70475086Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"618\",\"timestamp\":\"2022-08-12T13:30:04.704883009Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"619\",\"timestamp\":\"2022-08-12T13:30:04.704884973Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"620\",\"timestamp\":\"2022-08-12T13:30:04.704896004Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"621\",\"timestamp\":\"2022-08-12T13:30:04.704920169Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"622\",\"timestamp\":\"2022-08-12T13:30:04.704990135Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"623\",\"timestamp\":\"2022-08-12T13:30:04.704991597Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"624\",\"timestamp\":\"2022-08-12T13:30:04.705002334Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"625\",\"timestamp\":\"2022-08-12T13:30:04.705023982Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"626\",\"timestamp\":\"2022-08-12T13:30:04.705087983Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"627\",\"timestamp\":\"2022-08-12T13:30:04.705089826Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"628\",\"timestamp\":\"2022-08-12T13:30:04.705100979Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"629\",\"timestamp\":\"2022-08-12T13:30:04.70512312Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"630\",\"timestamp\":\"2022-08-12T13:30:04.705189122Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"631\",\"timestamp\":\"2022-08-12T13:30:04.705190673Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"632\",\"timestamp\":\"2022-08-12T13:30:04.705201969Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"633\",\"timestamp\":\"2022-08-12T13:30:04.705224357Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"634\",\"timestamp\":\"2022-08-12T13:30:04.705310603Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"635\",\"timestamp\":\"2022-08-12T13:30:04.705312048Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"636\",\"timestamp\":\"2022-08-12T13:30:04.705322383Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"637\",\"timestamp\":\"2022-08-12T13:30:04.70534587Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"638\",\"timestamp\":\"2022-08-12T13:30:04.705409513Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"639\",\"timestamp\":\"2022-08-12T13:30:04.705410943Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"640\",\"timestamp\":\"2022-08-12T13:30:04.705425433Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"641\",\"timestamp\":\"2022-08-12T13:30:04.70544809Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"642\",\"timestamp\":\"2022-08-12T13:30:04.705507883Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"643\",\"timestamp\":\"2022-08-12T13:30:04.705509746Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"644\",\"timestamp\":\"2022-08-12T13:30:04.705521201Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"645\",\"timestamp\":\"2022-08-12T13:30:04.705542864Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"646\",\"timestamp\":\"2022-08-12T13:30:04.705603098Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"647\",\"timestamp\":\"2022-08-12T13:30:04.70560443Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"648\",\"timestamp\":\"2022-08-12T13:30:04.705615912Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"649\",\"timestamp\":\"2022-08-12T13:30:04.705637946Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"650\",\"timestamp\":\"2022-08-12T13:30:04.705766838Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"651\",\"timestamp\":\"2022-08-12T13:30:04.70588959Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"652\",\"timestamp\":\"2022-08-12T13:30:04.705896459Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"653\",\"timestamp\":\"2022-08-12T13:30:04.705910605Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"654\",\"timestamp\":\"2022-08-12T13:30:04.706052465Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"655\",\"timestamp\":\"2022-08-12T13:30:04.706054195Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"656\",\"timestamp\":\"2022-08-12T13:30:04.706065743Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"657\",\"timestamp\":\"2022-08-12T13:30:04.706114943Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"658\",\"timestamp\":\"2022-08-12T13:30:04.706188344Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"659\",\"timestamp\":\"2022-08-12T13:30:04.706190134Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"660\",\"timestamp\":\"2022-08-12T13:30:04.706201666Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"661\",\"timestamp\":\"2022-08-12T13:30:04.706234783Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"662\",\"timestamp\":\"2022-08-12T13:30:04.706321398Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"663\",\"timestamp\":\"2022-08-12T13:30:04.70632295Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"664\",\"timestamp\":\"2022-08-12T13:30:04.706334209Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"665\",\"timestamp\":\"2022-08-12T13:30:04.706366095Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"666\",\"timestamp\":\"2022-08-12T13:30:04.706457669Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"667\",\"timestamp\":\"2022-08-12T13:30:04.706459107Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"668\",\"timestamp\":\"2022-08-12T13:30:04.706469863Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"669\",\"timestamp\":\"2022-08-12T13:30:04.706494218Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"670\",\"timestamp\":\"2022-08-12T13:30:04.706578986Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"671\",\"timestamp\":\"2022-08-12T13:30:04.706581809Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"672\",\"timestamp\":\"2022-08-12T13:30:04.70659846Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"673\",\"timestamp\":\"2022-08-12T13:30:04.706624482Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"674\",\"timestamp\":\"2022-08-12T13:30:04.706698446Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"675\",\"timestamp\":\"2022-08-12T13:30:04.706700361Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"676\",\"timestamp\":\"2022-08-12T13:30:04.706714027Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"677\",\"timestamp\":\"2022-08-12T13:30:04.706743676Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"678\",\"timestamp\":\"2022-08-12T13:30:04.706870599Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"679\",\"timestamp\":\"2022-08-12T13:30:04.706872584Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"680\",\"timestamp\":\"2022-08-12T13:30:04.7068853Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"681\",\"timestamp\":\"2022-08-12T13:30:04.706933435Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"682\",\"timestamp\":\"2022-08-12T13:30:04.707013541Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"683\",\"timestamp\":\"2022-08-12T13:30:04.707015012Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"684\",\"timestamp\":\"2022-08-12T13:30:04.707027844Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"685\",\"timestamp\":\"2022-08-12T13:30:04.707056491Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"686\",\"timestamp\":\"2022-08-12T13:30:04.707146544Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"687\",\"timestamp\":\"2022-08-12T13:30:04.707148249Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"688\",\"timestamp\":\"2022-08-12T13:30:04.707159028Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"689\",\"timestamp\":\"2022-08-12T13:30:04.707186468Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"690\",\"timestamp\":\"2022-08-12T13:30:04.707248597Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"691\",\"timestamp\":\"2022-08-12T13:30:04.707250262Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"692\",\"timestamp\":\"2022-08-12T13:30:04.707261454Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"693\",\"timestamp\":\"2022-08-12T13:30:04.707284654Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"694\",\"timestamp\":\"2022-08-12T13:30:04.707344038Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"695\",\"timestamp\":\"2022-08-12T13:30:04.70734543Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"696\",\"timestamp\":\"2022-08-12T13:30:04.707356688Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"697\",\"timestamp\":\"2022-08-12T13:30:04.707380004Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"698\",\"timestamp\":\"2022-08-12T13:30:04.707444475Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"699\",\"timestamp\":\"2022-08-12T13:30:04.707452512Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"700\",\"timestamp\":\"2022-08-12T13:30:04.707479109Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"701\",\"timestamp\":\"2022-08-12T13:30:04.707513104Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"702\",\"timestamp\":\"2022-08-12T13:30:04.707573415Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"703\",\"timestamp\":\"2022-08-12T13:30:04.70757489Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"704\",\"timestamp\":\"2022-08-12T13:30:04.707585715Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"705\",\"timestamp\":\"2022-08-12T13:30:04.707609484Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"706\",\"timestamp\":\"2022-08-12T13:30:04.707656123Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"707\",\"timestamp\":\"2022-08-12T13:30:04.707657633Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"708\",\"timestamp\":\"2022-08-12T13:30:04.707670601Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"709\",\"timestamp\":\"2022-08-12T13:30:04.707693525Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"710\",\"timestamp\":\"2022-08-12T13:30:04.707752155Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"711\",\"timestamp\":\"2022-08-12T13:30:04.707753711Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"712\",\"timestamp\":\"2022-08-12T13:30:04.707764721Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"713\",\"timestamp\":\"2022-08-12T13:30:04.707788294Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"714\",\"timestamp\":\"2022-08-12T13:30:04.707838814Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"715\",\"timestamp\":\"2022-08-12T13:30:04.707840132Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"716\",\"timestamp\":\"2022-08-12T13:30:04.707850656Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"717\",\"timestamp\":\"2022-08-12T13:30:04.707872226Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"718\",\"timestamp\":\"2022-08-12T13:30:04.707929428Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"719\",\"timestamp\":\"2022-08-12T13:30:04.707930885Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"720\",\"timestamp\":\"2022-08-12T13:30:04.70794376Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"721\",\"timestamp\":\"2022-08-12T13:30:04.707966601Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"722\",\"timestamp\":\"2022-08-12T13:30:04.708034262Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"723\",\"timestamp\":\"2022-08-12T13:30:04.708035589Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"724\",\"timestamp\":\"2022-08-12T13:30:04.708045668Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"725\",\"timestamp\":\"2022-08-12T13:30:04.708068205Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"726\",\"timestamp\":\"2022-08-12T13:30:04.708178754Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"727\",\"timestamp\":\"2022-08-12T13:30:04.708181443Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"728\",\"timestamp\":\"2022-08-12T13:30:04.708232434Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"729\",\"timestamp\":\"2022-08-12T13:30:04.708234571Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"730\",\"timestamp\":\"2022-08-12T13:30:04.708369392Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"731\",\"timestamp\":\"2022-08-12T13:30:04.708372512Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"732\",\"timestamp\":\"2022-08-12T13:30:04.708395368Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"733\",\"timestamp\":\"2022-08-12T13:30:04.708445942Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"734\",\"timestamp\":\"2022-08-12T13:30:04.708553658Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"735\",\"timestamp\":\"2022-08-12T13:30:04.708556006Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"736\",\"timestamp\":\"2022-08-12T13:30:04.708575383Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"737\",\"timestamp\":\"2022-08-12T13:30:04.708613754Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"738\",\"timestamp\":\"2022-08-12T13:30:04.708726841Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"739\",\"timestamp\":\"2022-08-12T13:30:04.708729311Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"740\",\"timestamp\":\"2022-08-12T13:30:04.708745908Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"741\",\"timestamp\":\"2022-08-12T13:30:04.708777182Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"742\",\"timestamp\":\"2022-08-12T13:30:04.708935272Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"743\",\"timestamp\":\"2022-08-12T13:30:04.708937344Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"744\",\"timestamp\":\"2022-08-12T13:30:04.708954836Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"745\",\"timestamp\":\"2022-08-12T13:30:04.709000071Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"746\",\"timestamp\":\"2022-08-12T13:30:04.70912482Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"747\",\"timestamp\":\"2022-08-12T13:30:04.709128254Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"748\",\"timestamp\":\"2022-08-12T13:30:04.709150148Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"749\",\"timestamp\":\"2022-08-12T13:30:04.709210662Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"750\",\"timestamp\":\"2022-08-12T13:30:04.709309652Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"751\",\"timestamp\":\"2022-08-12T13:30:04.709312103Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"752\",\"timestamp\":\"2022-08-12T13:30:04.709331811Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"753\",\"timestamp\":\"2022-08-12T13:30:04.709382885Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"754\",\"timestamp\":\"2022-08-12T13:30:04.709431308Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"755\",\"timestamp\":\"2022-08-12T13:30:04.709433057Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"756\",\"timestamp\":\"2022-08-12T13:30:04.709451889Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"757\",\"timestamp\":\"2022-08-12T13:30:04.709491313Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"758\",\"timestamp\":\"2022-08-12T13:30:04.709590523Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"759\",\"timestamp\":\"2022-08-12T13:30:04.709592009Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"760\",\"timestamp\":\"2022-08-12T13:30:04.709605244Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"761\",\"timestamp\":\"2022-08-12T13:30:04.709636937Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"762\",\"timestamp\":\"2022-08-12T13:30:04.709706196Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"763\",\"timestamp\":\"2022-08-12T13:30:04.709708061Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"764\",\"timestamp\":\"2022-08-12T13:30:04.709727055Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"765\",\"timestamp\":\"2022-08-12T13:30:04.709759537Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"766\",\"timestamp\":\"2022-08-12T13:30:04.709821448Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"767\",\"timestamp\":\"2022-08-12T13:30:04.709823104Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"768\",\"timestamp\":\"2022-08-12T13:30:04.709833983Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"769\",\"timestamp\":\"2022-08-12T13:30:04.709863828Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"770\",\"timestamp\":\"2022-08-12T13:30:04.709974918Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"771\",\"timestamp\":\"2022-08-12T13:30:04.709977426Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"772\",\"timestamp\":\"2022-08-12T13:30:04.709991337Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"773\",\"timestamp\":\"2022-08-12T13:30:04.710025274Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"774\",\"timestamp\":\"2022-08-12T13:30:04.71010215Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"775\",\"timestamp\":\"2022-08-12T13:30:04.710103771Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"776\",\"timestamp\":\"2022-08-12T13:30:04.710114715Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"777\",\"timestamp\":\"2022-08-12T13:30:04.71014817Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"778\",\"timestamp\":\"2022-08-12T13:30:04.710215554Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"779\",\"timestamp\":\"2022-08-12T13:30:04.710216994Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"780\",\"timestamp\":\"2022-08-12T13:30:04.710229487Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"781\",\"timestamp\":\"2022-08-12T13:30:04.710253338Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"782\",\"timestamp\":\"2022-08-12T13:30:04.710318654Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"783\",\"timestamp\":\"2022-08-12T13:30:04.710320067Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"784\",\"timestamp\":\"2022-08-12T13:30:04.710330402Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"785\",\"timestamp\":\"2022-08-12T13:30:04.710354084Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"786\",\"timestamp\":\"2022-08-12T13:30:04.710418256Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"787\",\"timestamp\":\"2022-08-12T13:30:04.710419756Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"788\",\"timestamp\":\"2022-08-12T13:30:04.710430434Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"789\",\"timestamp\":\"2022-08-12T13:30:04.710454041Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"790\",\"timestamp\":\"2022-08-12T13:30:04.710512365Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"791\",\"timestamp\":\"2022-08-12T13:30:04.710513768Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"792\",\"timestamp\":\"2022-08-12T13:30:04.710525787Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"793\",\"timestamp\":\"2022-08-12T13:30:04.710549626Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"794\",\"timestamp\":\"2022-08-12T13:30:04.710615114Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"795\",\"timestamp\":\"2022-08-12T13:30:04.710616478Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"796\",\"timestamp\":\"2022-08-12T13:30:04.710627446Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"797\",\"timestamp\":\"2022-08-12T13:30:04.710652112Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"798\",\"timestamp\":\"2022-08-12T13:30:04.710716468Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"799\",\"timestamp\":\"2022-08-12T13:30:04.710717858Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"800\",\"timestamp\":\"2022-08-12T13:30:04.710728396Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"801\",\"timestamp\":\"2022-08-12T13:30:04.710752871Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"802\",\"timestamp\":\"2022-08-12T13:30:04.710837134Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"803\",\"timestamp\":\"2022-08-12T13:30:04.710838506Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"804\",\"timestamp\":\"2022-08-12T13:30:04.710848881Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"805\",\"timestamp\":\"2022-08-12T13:30:04.7108824Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"806\",\"timestamp\":\"2022-08-12T13:30:04.710962594Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"807\",\"timestamp\":\"2022-08-12T13:30:04.710964116Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"808\",\"timestamp\":\"2022-08-12T13:30:04.710974224Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"809\",\"timestamp\":\"2022-08-12T13:30:04.710997471Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"810\",\"timestamp\":\"2022-08-12T13:30:04.711072504Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"811\",\"timestamp\":\"2022-08-12T13:30:04.711073919Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"812\",\"timestamp\":\"2022-08-12T13:30:04.71108442Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"813\",\"timestamp\":\"2022-08-12T13:30:04.71110896Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"814\",\"timestamp\":\"2022-08-12T13:30:04.711168002Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"815\",\"timestamp\":\"2022-08-12T13:30:04.7111694Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"816\",\"timestamp\":\"2022-08-12T13:30:04.711200056Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"817\",\"timestamp\":\"2022-08-12T13:30:04.711201407Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"818\",\"timestamp\":\"2022-08-12T13:30:04.711284343Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"819\",\"timestamp\":\"2022-08-12T13:30:04.711286559Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"820\",\"timestamp\":\"2022-08-12T13:30:04.711302987Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"821\",\"timestamp\":\"2022-08-12T13:30:04.711333257Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"822\",\"timestamp\":\"2022-08-12T13:30:04.711461684Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"823\",\"timestamp\":\"2022-08-12T13:30:04.711463275Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"824\",\"timestamp\":\"2022-08-12T13:30:04.711473831Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"825\",\"timestamp\":\"2022-08-12T13:30:04.711510722Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"826\",\"timestamp\":\"2022-08-12T13:30:04.711776112Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"827\",\"timestamp\":\"2022-08-12T13:30:04.711778088Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"828\",\"timestamp\":\"2022-08-12T13:30:04.711790555Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"829\",\"timestamp\":\"2022-08-12T13:30:04.711850892Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"830\",\"timestamp\":\"2022-08-12T13:30:04.711955921Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"831\",\"timestamp\":\"2022-08-12T13:30:04.711959057Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"832\",\"timestamp\":\"2022-08-12T13:30:04.711979285Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"833\",\"timestamp\":\"2022-08-12T13:30:04.712012517Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"834\",\"timestamp\":\"2022-08-12T13:30:04.712141797Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"835\",\"timestamp\":\"2022-08-12T13:30:04.712144941Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"836\",\"timestamp\":\"2022-08-12T13:30:04.712167389Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"837\",\"timestamp\":\"2022-08-12T13:30:04.71222609Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"838\",\"timestamp\":\"2022-08-12T13:30:04.712275903Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"839\",\"timestamp\":\"2022-08-12T13:30:04.71227873Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"840\",\"timestamp\":\"2022-08-12T13:30:04.712298541Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"841\",\"timestamp\":\"2022-08-12T13:30:04.712346086Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"844\",\"timestamp\":\"2022-08-12T13:30:04.712432623Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"842\",\"timestamp\":\"2022-08-12T13:30:04.712417798Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"845\",\"timestamp\":\"2022-08-12T13:30:04.712475176Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"843\",\"timestamp\":\"2022-08-12T13:30:04.712419615Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"846\",\"timestamp\":\"2022-08-12T13:30:04.712541331Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"847\",\"timestamp\":\"2022-08-12T13:30:04.712544343Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"848\",\"timestamp\":\"2022-08-12T13:30:04.712565343Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"849\",\"timestamp\":\"2022-08-12T13:30:04.71260141Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"850\",\"timestamp\":\"2022-08-12T13:30:04.712679533Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"851\",\"timestamp\":\"2022-08-12T13:30:04.712682047Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"852\",\"timestamp\":\"2022-08-12T13:30:04.71269993Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"853\",\"timestamp\":\"2022-08-12T13:30:04.712744578Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"854\",\"timestamp\":\"2022-08-12T13:30:04.712832526Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"855\",\"timestamp\":\"2022-08-12T13:30:04.712835083Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"856\",\"timestamp\":\"2022-08-12T13:30:04.712855748Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"857\",\"timestamp\":\"2022-08-12T13:30:04.712901117Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"858\",\"timestamp\":\"2022-08-12T13:30:04.712948278Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"859\",\"timestamp\":\"2022-08-12T13:30:04.712950652Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"860\",\"timestamp\":\"2022-08-12T13:30:04.712972108Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"861\",\"timestamp\":\"2022-08-12T13:30:04.713013007Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"862\",\"timestamp\":\"2022-08-12T13:30:04.713086755Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"863\",\"timestamp\":\"2022-08-12T13:30:04.713095643Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"864\",\"timestamp\":\"2022-08-12T13:30:04.713124847Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"865\",\"timestamp\":\"2022-08-12T13:30:04.713193224Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"868\",\"timestamp\":\"2022-08-12T13:30:04.713242747Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"866\",\"timestamp\":\"2022-08-12T13:30:04.713230415Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"869\",\"timestamp\":\"2022-08-12T13:30:04.713273546Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"867\",\"timestamp\":\"2022-08-12T13:30:04.713231837Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"870\",\"timestamp\":\"2022-08-12T13:30:04.713374227Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"871\",\"timestamp\":\"2022-08-12T13:30:04.713376026Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"876\",\"timestamp\":\"2022-08-12T13:30:04.713494103Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"877\",\"timestamp\":\"2022-08-12T13:30:04.713560486Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"878\",\"timestamp\":\"2022-08-12T13:30:04.713617852Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"872\",\"timestamp\":\"2022-08-12T13:30:04.71338764Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"879\",\"timestamp\":\"2022-08-12T13:30:04.713619264Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"873\",\"timestamp\":\"2022-08-12T13:30:04.71343859Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"874\",\"timestamp\":\"2022-08-12T13:30:04.713480681Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"875\",\"timestamp\":\"2022-08-12T13:30:04.713482158Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"880\",\"timestamp\":\"2022-08-12T13:30:04.713635908Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"881\",\"timestamp\":\"2022-08-12T13:30:04.713666749Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"882\",\"timestamp\":\"2022-08-12T13:30:04.713737208Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"883\",\"timestamp\":\"2022-08-12T13:30:04.713738675Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"884\",\"timestamp\":\"2022-08-12T13:30:04.713749906Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"885\",\"timestamp\":\"2022-08-12T13:30:04.713776348Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"886\",\"timestamp\":\"2022-08-12T13:30:04.713842556Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"887\",\"timestamp\":\"2022-08-12T13:30:04.713844333Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"888\",\"timestamp\":\"2022-08-12T13:30:04.713856017Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"889\",\"timestamp\":\"2022-08-12T13:30:04.713918858Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"890\",\"timestamp\":\"2022-08-12T13:30:04.713957075Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"891\",\"timestamp\":\"2022-08-12T13:30:04.713958725Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"892\",\"timestamp\":\"2022-08-12T13:30:04.71397766Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"893\",\"timestamp\":\"2022-08-12T13:30:04.713997861Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"894\",\"timestamp\":\"2022-08-12T13:30:04.714068363Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"895\",\"timestamp\":\"2022-08-12T13:30:04.714069854Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"904\",\"timestamp\":\"2022-08-12T13:30:04.71441155Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"905\",\"timestamp\":\"2022-08-12T13:30:04.714443146Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"906\",\"timestamp\":\"2022-08-12T13:30:04.714526487Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"907\",\"timestamp\":\"2022-08-12T13:30:04.71452801Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"900\",\"timestamp\":\"2022-08-12T13:30:04.714259517Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"901\",\"timestamp\":\"2022-08-12T13:30:04.71431771Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"896\",\"timestamp\":\"2022-08-12T13:30:04.714081946Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"902\",\"timestamp\":\"2022-08-12T13:30:04.714398763Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"897\",\"timestamp\":\"2022-08-12T13:30:04.714131856Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"903\",\"timestamp\":\"2022-08-12T13:30:04.714400278Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"898\",\"timestamp\":\"2022-08-12T13:30:04.714221145Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"899\",\"timestamp\":\"2022-08-12T13:30:04.714222816Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"908\",\"timestamp\":\"2022-08-12T13:30:04.714539876Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"909\",\"timestamp\":\"2022-08-12T13:30:04.714579616Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"910\",\"timestamp\":\"2022-08-12T13:30:04.714648781Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"911\",\"timestamp\":\"2022-08-12T13:30:04.714650352Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"912\",\"timestamp\":\"2022-08-12T13:30:04.714661169Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"913\",\"timestamp\":\"2022-08-12T13:30:04.714687341Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"914\",\"timestamp\":\"2022-08-12T13:30:04.714752976Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"915\",\"timestamp\":\"2022-08-12T13:30:04.71475437Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"916\",\"timestamp\":\"2022-08-12T13:30:04.714766934Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"917\",\"timestamp\":\"2022-08-12T13:30:04.714791959Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"918\",\"timestamp\":\"2022-08-12T13:30:04.714855719Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"919\",\"timestamp\":\"2022-08-12T13:30:04.714857136Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"924\",\"timestamp\":\"2022-08-12T13:30:04.714966515Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"920\",\"timestamp\":\"2022-08-12T13:30:04.71486945Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"925\",\"timestamp\":\"2022-08-12T13:30:04.714989623Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"921\",\"timestamp\":\"2022-08-12T13:30:04.714897783Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"926\",\"timestamp\":\"2022-08-12T13:30:04.715051119Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"922\",\"timestamp\":\"2022-08-12T13:30:04.714953405Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"927\",\"timestamp\":\"2022-08-12T13:30:04.715052519Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"923\",\"timestamp\":\"2022-08-12T13:30:04.714954731Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"928\",\"timestamp\":\"2022-08-12T13:30:04.715063486Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"929\",\"timestamp\":\"2022-08-12T13:30:04.715086245Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"930\",\"timestamp\":\"2022-08-12T13:30:04.71514682Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"931\",\"timestamp\":\"2022-08-12T13:30:04.715208913Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"932\",\"timestamp\":\"2022-08-12T13:30:04.715309406Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"933\",\"timestamp\":\"2022-08-12T13:30:04.715319793Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"934\",\"timestamp\":\"2022-08-12T13:30:04.715330232Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"935\",\"timestamp\":\"2022-08-12T13:30:04.715355303Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"936\",\"timestamp\":\"2022-08-12T13:30:04.715367929Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"937\",\"timestamp\":\"2022-08-12T13:30:04.715388934Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"938\",\"timestamp\":\"2022-08-12T13:30:04.715536443Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"939\",\"timestamp\":\"2022-08-12T13:30:04.715537919Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"940\",\"timestamp\":\"2022-08-12T13:30:04.715548611Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"941\",\"timestamp\":\"2022-08-12T13:30:04.715578812Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"942\",\"timestamp\":\"2022-08-12T13:30:04.715644144Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"943\",\"timestamp\":\"2022-08-12T13:30:04.71564598Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"944\",\"timestamp\":\"2022-08-12T13:30:04.715657848Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"945\",\"timestamp\":\"2022-08-12T13:30:04.715697271Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"946\",\"timestamp\":\"2022-08-12T13:30:04.715762211Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"947\",\"timestamp\":\"2022-08-12T13:30:04.715764594Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"952\",\"timestamp\":\"2022-08-12T13:30:04.715951386Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"953\",\"timestamp\":\"2022-08-12T13:30:04.715985421Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"954\",\"timestamp\":\"2022-08-12T13:30:04.71606346Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"955\",\"timestamp\":\"2022-08-12T13:30:04.716065057Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"948\",\"timestamp\":\"2022-08-12T13:30:04.715784626Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"949\",\"timestamp\":\"2022-08-12T13:30:04.715819265Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"950\",\"timestamp\":\"2022-08-12T13:30:04.715926101Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"951\",\"timestamp\":\"2022-08-12T13:30:04.715928999Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"956\",\"timestamp\":\"2022-08-12T13:30:04.716078469Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"957\",\"timestamp\":\"2022-08-12T13:30:04.716105203Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"958\",\"timestamp\":\"2022-08-12T13:30:04.716186805Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"959\",\"timestamp\":\"2022-08-12T13:30:04.716189249Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"960\",\"timestamp\":\"2022-08-12T13:30:04.716209061Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"961\",\"timestamp\":\"2022-08-12T13:30:04.716261576Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"962\",\"timestamp\":\"2022-08-12T13:30:04.716300495Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"963\",\"timestamp\":\"2022-08-12T13:30:04.716301971Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"964\",\"timestamp\":\"2022-08-12T13:30:04.716312989Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"965\",\"timestamp\":\"2022-08-12T13:30:04.716348322Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"966\",\"timestamp\":\"2022-08-12T13:30:04.716423956Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"967\",\"timestamp\":\"2022-08-12T13:30:04.716426315Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"968\",\"timestamp\":\"2022-08-12T13:30:04.716447423Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"969\",\"timestamp\":\"2022-08-12T13:30:04.716509994Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"970\",\"timestamp\":\"2022-08-12T13:30:04.71658427Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"971\",\"timestamp\":\"2022-08-12T13:30:04.716585802Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"972\",\"timestamp\":\"2022-08-12T13:30:04.716597559Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"973\",\"timestamp\":\"2022-08-12T13:30:04.716640621Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"974\",\"timestamp\":\"2022-08-12T13:30:04.716699467Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"975\",\"timestamp\":\"2022-08-12T13:30:04.716702214Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"976\",\"timestamp\":\"2022-08-12T13:30:04.716731172Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"977\",\"timestamp\":\"2022-08-12T13:30:04.716795598Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"978\",\"timestamp\":\"2022-08-12T13:30:04.716888316Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"979\",\"timestamp\":\"2022-08-12T13:30:04.716891115Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"980\",\"timestamp\":\"2022-08-12T13:30:04.716912043Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"981\",\"timestamp\":\"2022-08-12T13:30:04.716949806Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"982\",\"timestamp\":\"2022-08-12T13:30:04.717044262Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"983\",\"timestamp\":\"2022-08-12T13:30:04.717046304Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"984\",\"timestamp\":\"2022-08-12T13:30:04.717058535Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"985\",\"timestamp\":\"2022-08-12T13:30:04.717089878Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"988\",\"timestamp\":\"2022-08-12T13:30:04.717224621Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"986\",\"timestamp\":\"2022-08-12T13:30:04.717201882Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"989\",\"timestamp\":\"2022-08-12T13:30:04.717255035Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"987\",\"timestamp\":\"2022-08-12T13:30:04.71720484Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"990\",\"timestamp\":\"2022-08-12T13:30:04.717361332Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"991\",\"timestamp\":\"2022-08-12T13:30:04.717363063Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"992\",\"timestamp\":\"2022-08-12T13:30:04.717374202Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"993\",\"timestamp\":\"2022-08-12T13:30:04.717422883Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"994\",\"timestamp\":\"2022-08-12T13:30:04.717540795Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"995\",\"timestamp\":\"2022-08-12T13:30:04.717542899Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"996\",\"timestamp\":\"2022-08-12T13:30:04.717559179Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1000\",\"timestamp\":\"2022-08-12T13:30:04.71770325Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"997\",\"timestamp\":\"2022-08-12T13:30:04.717602673Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1001\",\"timestamp\":\"2022-08-12T13:30:04.717748364Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"998\",\"timestamp\":\"2022-08-12T13:30:04.717689994Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1002\",\"timestamp\":\"2022-08-12T13:30:04.717810876Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"999\",\"timestamp\":\"2022-08-12T13:30:04.717691639Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1003\",\"timestamp\":\"2022-08-12T13:30:04.717813824Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1004\",\"timestamp\":\"2022-08-12T13:30:04.717825579Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1005\",\"timestamp\":\"2022-08-12T13:30:04.717850867Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1006\",\"timestamp\":\"2022-08-12T13:30:04.717913556Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1007\",\"timestamp\":\"2022-08-12T13:30:04.717915015Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1008\",\"timestamp\":\"2022-08-12T13:30:04.717925497Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1009\",\"timestamp\":\"2022-08-12T13:30:04.717947864Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1010\",\"timestamp\":\"2022-08-12T13:30:04.718040632Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1011\",\"timestamp\":\"2022-08-12T13:30:04.718042702Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1012\",\"timestamp\":\"2022-08-12T13:30:04.718098444Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1013\",\"timestamp\":\"2022-08-12T13:30:04.718151358Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1014\",\"timestamp\":\"2022-08-12T13:30:04.718234563Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1015\",\"timestamp\":\"2022-08-12T13:30:04.718236201Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1016\",\"timestamp\":\"2022-08-12T13:30:04.718251774Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1017\",\"timestamp\":\"2022-08-12T13:30:04.718279106Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1018\",\"timestamp\":\"2022-08-12T13:30:04.71836407Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1019\",\"timestamp\":\"2022-08-12T13:30:04.71836553Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1020\",\"timestamp\":\"2022-08-12T13:30:04.718378678Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1021\",\"timestamp\":\"2022-08-12T13:30:04.718403592Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1022\",\"timestamp\":\"2022-08-12T13:30:04.718492334Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1023\",\"timestamp\":\"2022-08-12T13:30:04.718494063Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1024\",\"timestamp\":\"2022-08-12T13:30:04.718505552Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1025\",\"timestamp\":\"2022-08-12T13:30:04.718558491Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1026\",\"timestamp\":\"2022-08-12T13:30:04.718634674Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1027\",\"timestamp\":\"2022-08-12T13:30:04.71863619Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1028\",\"timestamp\":\"2022-08-12T13:30:04.718647076Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1029\",\"timestamp\":\"2022-08-12T13:30:04.718672497Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1030\",\"timestamp\":\"2022-08-12T13:30:04.718735921Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1031\",\"timestamp\":\"2022-08-12T13:30:04.718737498Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1032\",\"timestamp\":\"2022-08-12T13:30:04.718757782Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1033\",\"timestamp\":\"2022-08-12T13:30:04.71878533Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1034\",\"timestamp\":\"2022-08-12T13:30:04.718850074Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1035\",\"timestamp\":\"2022-08-12T13:30:04.718851545Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1036\",\"timestamp\":\"2022-08-12T13:30:04.718862618Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1037\",\"timestamp\":\"2022-08-12T13:30:04.718888289Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1038\",\"timestamp\":\"2022-08-12T13:30:04.71899266Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1039\",\"timestamp\":\"2022-08-12T13:30:04.718994119Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1040\",\"timestamp\":\"2022-08-12T13:30:04.719006465Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1041\",\"timestamp\":\"2022-08-12T13:30:04.719034067Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1042\",\"timestamp\":\"2022-08-12T13:30:04.719117434Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1043\",\"timestamp\":\"2022-08-12T13:30:04.719118934Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1044\",\"timestamp\":\"2022-08-12T13:30:04.719129992Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1045\",\"timestamp\":\"2022-08-12T13:30:04.719154181Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1046\",\"timestamp\":\"2022-08-12T13:30:04.719220595Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1047\",\"timestamp\":\"2022-08-12T13:30:04.719222004Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1048\",\"timestamp\":\"2022-08-12T13:30:04.71923287Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1049\",\"timestamp\":\"2022-08-12T13:30:04.719257141Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1050\",\"timestamp\":\"2022-08-12T13:30:04.719305938Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1051\",\"timestamp\":\"2022-08-12T13:30:04.719307428Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1052\",\"timestamp\":\"2022-08-12T13:30:04.719320222Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1053\",\"timestamp\":\"2022-08-12T13:30:04.719344568Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1054\",\"timestamp\":\"2022-08-12T13:30:04.719394472Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1055\",\"timestamp\":\"2022-08-12T13:30:04.719395892Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1056\",\"timestamp\":\"2022-08-12T13:30:04.719406265Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1057\",\"timestamp\":\"2022-08-12T13:30:04.719435252Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1058\",\"timestamp\":\"2022-08-12T13:30:04.719482663Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1059\",\"timestamp\":\"2022-08-12T13:30:04.719484978Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1060\",\"timestamp\":\"2022-08-12T13:30:04.719500193Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1061\",\"timestamp\":\"2022-08-12T13:30:04.719544044Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1062\",\"timestamp\":\"2022-08-12T13:30:04.719601622Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1063\",\"timestamp\":\"2022-08-12T13:30:04.719603628Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1064\",\"timestamp\":\"2022-08-12T13:30:04.719620626Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1065\",\"timestamp\":\"2022-08-12T13:30:04.71966462Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1066\",\"timestamp\":\"2022-08-12T13:30:04.719718699Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1067\",\"timestamp\":\"2022-08-12T13:30:04.719720137Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1068\",\"timestamp\":\"2022-08-12T13:30:04.719730749Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1069\",\"timestamp\":\"2022-08-12T13:30:04.719755893Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1070\",\"timestamp\":\"2022-08-12T13:30:04.719820477Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1071\",\"timestamp\":\"2022-08-12T13:30:04.719823913Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1072\",\"timestamp\":\"2022-08-12T13:30:04.719851384Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1073\",\"timestamp\":\"2022-08-12T13:30:04.719900268Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1074\",\"timestamp\":\"2022-08-12T13:30:04.719989798Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1075\",\"timestamp\":\"2022-08-12T13:30:04.719991293Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1076\",\"timestamp\":\"2022-08-12T13:30:04.720002148Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1080\",\"timestamp\":\"2022-08-12T13:30:04.720109938Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1077\",\"timestamp\":\"2022-08-12T13:30:04.720028646Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1081\",\"timestamp\":\"2022-08-12T13:30:04.720135214Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1078\",\"timestamp\":\"2022-08-12T13:30:04.72009678Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1082\",\"timestamp\":\"2022-08-12T13:30:04.720233666Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1079\",\"timestamp\":\"2022-08-12T13:30:04.720098245Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1083\",\"timestamp\":\"2022-08-12T13:30:04.720235717Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1084\",\"timestamp\":\"2022-08-12T13:30:04.720250266Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1085\",\"timestamp\":\"2022-08-12T13:30:04.72029262Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1086\",\"timestamp\":\"2022-08-12T13:30:04.720355963Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1087\",\"timestamp\":\"2022-08-12T13:30:04.720357702Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1092\",\"timestamp\":\"2022-08-12T13:30:04.720491846Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1093\",\"timestamp\":\"2022-08-12T13:30:04.720522913Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1088\",\"timestamp\":\"2022-08-12T13:30:04.720372571Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1094\",\"timestamp\":\"2022-08-12T13:30:04.720582715Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1095\",\"timestamp\":\"2022-08-12T13:30:04.720584485Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1089\",\"timestamp\":\"2022-08-12T13:30:04.720410812Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1090\",\"timestamp\":\"2022-08-12T13:30:04.720478491Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1091\",\"timestamp\":\"2022-08-12T13:30:04.720480218Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1096\",\"timestamp\":\"2022-08-12T13:30:04.720598203Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1097\",\"timestamp\":\"2022-08-12T13:30:04.720626808Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1098\",\"timestamp\":\"2022-08-12T13:30:04.72067591Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1099\",\"timestamp\":\"2022-08-12T13:30:04.720677401Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1100\",\"timestamp\":\"2022-08-12T13:30:04.720690184Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1101\",\"timestamp\":\"2022-08-12T13:30:04.720715219Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1102\",\"timestamp\":\"2022-08-12T13:30:04.720769041Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1103\",\"timestamp\":\"2022-08-12T13:30:04.720770757Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1104\",\"timestamp\":\"2022-08-12T13:30:04.720805757Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1105\",\"timestamp\":\"2022-08-12T13:30:04.72083091Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1106\",\"timestamp\":\"2022-08-12T13:30:04.720910654Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1107\",\"timestamp\":\"2022-08-12T13:30:04.720912234Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1108\",\"timestamp\":\"2022-08-12T13:30:04.720922603Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1109\",\"timestamp\":\"2022-08-12T13:30:04.720961357Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1110\",\"timestamp\":\"2022-08-12T13:30:04.721056935Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1112\",\"timestamp\":\"2022-08-12T13:30:04.721082107Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1111\",\"timestamp\":\"2022-08-12T13:30:04.721060116Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1113\",\"timestamp\":\"2022-08-12T13:30:04.721109374Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1114\",\"timestamp\":\"2022-08-12T13:30:04.721185212Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1115\",\"timestamp\":\"2022-08-12T13:30:04.721186698Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1116\",\"timestamp\":\"2022-08-12T13:30:04.721197567Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1117\",\"timestamp\":\"2022-08-12T13:30:04.721230527Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1118\",\"timestamp\":\"2022-08-12T13:30:04.72128482Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1119\",\"timestamp\":\"2022-08-12T13:30:04.721286311Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1120\",\"timestamp\":\"2022-08-12T13:30:04.721296783Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1121\",\"timestamp\":\"2022-08-12T13:30:04.721330626Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1122\",\"timestamp\":\"2022-08-12T13:30:04.721374349Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1123\",\"timestamp\":\"2022-08-12T13:30:04.721375723Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1124\",\"timestamp\":\"2022-08-12T13:30:04.721386016Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1125\",\"timestamp\":\"2022-08-12T13:30:04.721417245Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1126\",\"timestamp\":\"2022-08-12T13:30:04.721465258Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1127\",\"timestamp\":\"2022-08-12T13:30:04.721466618Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1128\",\"timestamp\":\"2022-08-12T13:30:04.721476998Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1129\",\"timestamp\":\"2022-08-12T13:30:04.721507429Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1130\",\"timestamp\":\"2022-08-12T13:30:04.721567582Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1131\",\"timestamp\":\"2022-08-12T13:30:04.721571813Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1132\",\"timestamp\":\"2022-08-12T13:30:04.721595411Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1133\",\"timestamp\":\"2022-08-12T13:30:04.721642744Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1134\",\"timestamp\":\"2022-08-12T13:30:04.72168199Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1135\",\"timestamp\":\"2022-08-12T13:30:04.721683526Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1136\",\"timestamp\":\"2022-08-12T13:30:04.721699658Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1137\",\"timestamp\":\"2022-08-12T13:30:04.721757707Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1138\",\"timestamp\":\"2022-08-12T13:30:04.7218142Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1139\",\"timestamp\":\"2022-08-12T13:30:04.721815936Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1140\",\"timestamp\":\"2022-08-12T13:30:04.721827629Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1141\",\"timestamp\":\"2022-08-12T13:30:04.721857375Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1142\",\"timestamp\":\"2022-08-12T13:30:04.72193039Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1143\",\"timestamp\":\"2022-08-12T13:30:04.721932146Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1144\",\"timestamp\":\"2022-08-12T13:30:04.721943453Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1148\",\"timestamp\":\"2022-08-12T13:30:04.722041659Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1145\",\"timestamp\":\"2022-08-12T13:30:04.721970231Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1149\",\"timestamp\":\"2022-08-12T13:30:04.722070899Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1146\",\"timestamp\":\"2022-08-12T13:30:04.722029146Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1150\",\"timestamp\":\"2022-08-12T13:30:04.722121361Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1147\",\"timestamp\":\"2022-08-12T13:30:04.72203073Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1151\",\"timestamp\":\"2022-08-12T13:30:04.722123262Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1152\",\"timestamp\":\"2022-08-12T13:30:04.722137804Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1153\",\"timestamp\":\"2022-08-12T13:30:04.722165053Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1154\",\"timestamp\":\"2022-08-12T13:30:04.722202637Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1155\",\"timestamp\":\"2022-08-12T13:30:04.722204357Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1156\",\"timestamp\":\"2022-08-12T13:30:04.722218697Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1157\",\"timestamp\":\"2022-08-12T13:30:04.722248043Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1158\",\"timestamp\":\"2022-08-12T13:30:04.722323697Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1159\",\"timestamp\":\"2022-08-12T13:30:04.722326559Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1160\",\"timestamp\":\"2022-08-12T13:30:04.722346125Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1161\",\"timestamp\":\"2022-08-12T13:30:04.72239747Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1162\",\"timestamp\":\"2022-08-12T13:30:04.722428438Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1163\",\"timestamp\":\"2022-08-12T13:30:04.722430372Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1164\",\"timestamp\":\"2022-08-12T13:30:04.722444225Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1165\",\"timestamp\":\"2022-08-12T13:30:04.722477559Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1166\",\"timestamp\":\"2022-08-12T13:30:04.722545503Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1167\",\"timestamp\":\"2022-08-12T13:30:04.722548106Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1168\",\"timestamp\":\"2022-08-12T13:30:04.722567527Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1169\",\"timestamp\":\"2022-08-12T13:30:04.722603043Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1170\",\"timestamp\":\"2022-08-12T13:30:04.722658737Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1171\",\"timestamp\":\"2022-08-12T13:30:04.72266028Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1172\",\"timestamp\":\"2022-08-12T13:30:04.722671179Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1173\",\"timestamp\":\"2022-08-12T13:30:04.72270491Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1174\",\"timestamp\":\"2022-08-12T13:30:04.722761636Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1175\",\"timestamp\":\"2022-08-12T13:30:04.722841361Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1176\",\"timestamp\":\"2022-08-12T13:30:04.722842974Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1177\",\"timestamp\":\"2022-08-12T13:30:04.722854654Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1178\",\"timestamp\":\"2022-08-12T13:30:04.722880331Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1181\",\"timestamp\":\"2022-08-12T13:30:04.722937624Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1179\",\"timestamp\":\"2022-08-12T13:30:04.722925764Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1182\",\"timestamp\":\"2022-08-12T13:30:04.722961927Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1180\",\"timestamp\":\"2022-08-12T13:30:04.722927287Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1183\",\"timestamp\":\"2022-08-12T13:30:04.723009558Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1184\",\"timestamp\":\"2022-08-12T13:30:04.723010906Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1185\",\"timestamp\":\"2022-08-12T13:30:04.72302468Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1186\",\"timestamp\":\"2022-08-12T13:30:04.7230504Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1187\",\"timestamp\":\"2022-08-12T13:30:04.72309843Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1188\",\"timestamp\":\"2022-08-12T13:30:04.723099875Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1189\",\"timestamp\":\"2022-08-12T13:30:04.723110268Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1190\",\"timestamp\":\"2022-08-12T13:30:04.723134788Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1191\",\"timestamp\":\"2022-08-12T13:30:04.72318872Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1192\",\"timestamp\":\"2022-08-12T13:30:04.723190113Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1193\",\"timestamp\":\"2022-08-12T13:30:04.72320054Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1194\",\"timestamp\":\"2022-08-12T13:30:04.72322426Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1195\",\"timestamp\":\"2022-08-12T13:30:04.72327904Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1196\",\"timestamp\":\"2022-08-12T13:30:04.723280423Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1197\",\"timestamp\":\"2022-08-12T13:30:04.723290803Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1198\",\"timestamp\":\"2022-08-12T13:30:04.723314474Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1199\",\"timestamp\":\"2022-08-12T13:30:04.723369582Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1200\",\"timestamp\":\"2022-08-12T13:30:04.723370982Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1201\",\"timestamp\":\"2022-08-12T13:30:04.723381446Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1202\",\"timestamp\":\"2022-08-12T13:30:04.723405137Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1203\",\"timestamp\":\"2022-08-12T13:30:04.723461813Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1204\",\"timestamp\":\"2022-08-12T13:30:04.723463173Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1205\",\"timestamp\":\"2022-08-12T13:30:04.723473901Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1206\",\"timestamp\":\"2022-08-12T13:30:04.723497556Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1207\",\"timestamp\":\"2022-08-12T13:30:04.723552536Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1208\",\"timestamp\":\"2022-08-12T13:30:04.723554122Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1209\",\"timestamp\":\"2022-08-12T13:30:04.723565649Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1210\",\"timestamp\":\"2022-08-12T13:30:04.723589159Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1211\",\"timestamp\":\"2022-08-12T13:30:04.723642396Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1212\",\"timestamp\":\"2022-08-12T13:30:04.723643789Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1213\",\"timestamp\":\"2022-08-12T13:30:04.723707718Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1214\",\"timestamp\":\"2022-08-12T13:30:04.723784035Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1215\",\"timestamp\":\"2022-08-12T13:30:04.723812348Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1216\",\"timestamp\":\"2022-08-12T13:30:04.72381381Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1217\",\"timestamp\":\"2022-08-12T13:30:04.723825329Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1218\",\"timestamp\":\"2022-08-12T13:30:04.72385155Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1219\",\"timestamp\":\"2022-08-12T13:30:04.723901202Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1220\",\"timestamp\":\"2022-08-12T13:30:04.723902573Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1221\",\"timestamp\":\"2022-08-12T13:30:04.723918403Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1222\",\"timestamp\":\"2022-08-12T13:30:04.723941755Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1223\",\"timestamp\":\"2022-08-12T13:30:04.723991497Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1224\",\"timestamp\":\"2022-08-12T13:30:04.723993912Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1225\",\"timestamp\":\"2022-08-12T13:30:04.724008957Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1226\",\"timestamp\":\"2022-08-12T13:30:04.724053334Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1227\",\"timestamp\":\"2022-08-12T13:30:04.724109193Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1228\",\"timestamp\":\"2022-08-12T13:30:04.724110657Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1229\",\"timestamp\":\"2022-08-12T13:30:04.72412113Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1230\",\"timestamp\":\"2022-08-12T13:30:04.724146191Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1231\",\"timestamp\":\"2022-08-12T13:30:04.724213719Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1232\",\"timestamp\":\"2022-08-12T13:30:04.724215244Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1233\",\"timestamp\":\"2022-08-12T13:30:04.724226742Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1234\",\"timestamp\":\"2022-08-12T13:30:04.724254342Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1235\",\"timestamp\":\"2022-08-12T13:30:04.724348093Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1236\",\"timestamp\":\"2022-08-12T13:30:04.724349915Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1237\",\"timestamp\":\"2022-08-12T13:30:04.724361177Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1238\",\"timestamp\":\"2022-08-12T13:30:04.724397888Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1239\",\"timestamp\":\"2022-08-12T13:30:04.72445394Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1240\",\"timestamp\":\"2022-08-12T13:30:04.724455777Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1241\",\"timestamp\":\"2022-08-12T13:30:04.724466161Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1242\",\"timestamp\":\"2022-08-12T13:30:04.724517422Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1243\",\"timestamp\":\"2022-08-12T13:30:04.724553559Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1244\",\"timestamp\":\"2022-08-12T13:30:04.724556403Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1245\",\"timestamp\":\"2022-08-12T13:30:04.724567528Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1246\",\"timestamp\":\"2022-08-12T13:30:04.724598314Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1247\",\"timestamp\":\"2022-08-12T13:30:04.724649254Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1248\",\"timestamp\":\"2022-08-12T13:30:04.724650763Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1249\",\"timestamp\":\"2022-08-12T13:30:04.724663703Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1250\",\"timestamp\":\"2022-08-12T13:30:04.724694939Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1251\",\"timestamp\":\"2022-08-12T13:30:04.724744175Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1252\",\"timestamp\":\"2022-08-12T13:30:04.724745563Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1253\",\"timestamp\":\"2022-08-12T13:30:04.724756505Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1254\",\"timestamp\":\"2022-08-12T13:30:04.724808177Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1255\",\"timestamp\":\"2022-08-12T13:30:04.724851439Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1256\",\"timestamp\":\"2022-08-12T13:30:04.724852945Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1257\",\"timestamp\":\"2022-08-12T13:30:04.724863212Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1258\",\"timestamp\":\"2022-08-12T13:30:04.72489549Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1259\",\"timestamp\":\"2022-08-12T13:30:04.72494261Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1260\",\"timestamp\":\"2022-08-12T13:30:04.724944021Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1261\",\"timestamp\":\"2022-08-12T13:30:04.724954912Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1262\",\"timestamp\":\"2022-08-12T13:30:04.724985556Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1263\",\"timestamp\":\"2022-08-12T13:30:04.725040078Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1264\",\"timestamp\":\"2022-08-12T13:30:04.725041609Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1265\",\"timestamp\":\"2022-08-12T13:30:04.725052849Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1266\",\"timestamp\":\"2022-08-12T13:30:04.725082432Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1267\",\"timestamp\":\"2022-08-12T13:30:04.725138728Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1268\",\"timestamp\":\"2022-08-12T13:30:04.725140187Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1269\",\"timestamp\":\"2022-08-12T13:30:04.725151372Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1270\",\"timestamp\":\"2022-08-12T13:30:04.725175776Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1271\",\"timestamp\":\"2022-08-12T13:30:04.725225587Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1272\",\"timestamp\":\"2022-08-12T13:30:04.725226994Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1273\",\"timestamp\":\"2022-08-12T13:30:04.725237458Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1274\",\"timestamp\":\"2022-08-12T13:30:04.725261546Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1275\",\"timestamp\":\"2022-08-12T13:30:04.725323126Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1276\",\"timestamp\":\"2022-08-12T13:30:04.725324729Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1277\",\"timestamp\":\"2022-08-12T13:30:04.725337708Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1278\",\"timestamp\":\"2022-08-12T13:30:04.725361718Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1279\",\"timestamp\":\"2022-08-12T13:30:04.725421275Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1280\",\"timestamp\":\"2022-08-12T13:30:04.725422632Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1281\",\"timestamp\":\"2022-08-12T13:30:04.725432696Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1282\",\"timestamp\":\"2022-08-12T13:30:04.725457678Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1283\",\"timestamp\":\"2022-08-12T13:30:04.72551114Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1284\",\"timestamp\":\"2022-08-12T13:30:04.725512574Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1285\",\"timestamp\":\"2022-08-12T13:30:04.725523899Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1286\",\"timestamp\":\"2022-08-12T13:30:04.725547565Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1287\",\"timestamp\":\"2022-08-12T13:30:04.725594163Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1288\",\"timestamp\":\"2022-08-12T13:30:04.725595469Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1289\",\"timestamp\":\"2022-08-12T13:30:04.725605873Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1290\",\"timestamp\":\"2022-08-12T13:30:04.725632579Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1291\",\"timestamp\":\"2022-08-12T13:30:04.725677381Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1292\",\"timestamp\":\"2022-08-12T13:30:04.725678797Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1293\",\"timestamp\":\"2022-08-12T13:30:04.725689297Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1294\",\"timestamp\":\"2022-08-12T13:30:04.725713233Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1295\",\"timestamp\":\"2022-08-12T13:30:04.725772885Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1296\",\"timestamp\":\"2022-08-12T13:30:04.725774249Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1297\",\"timestamp\":\"2022-08-12T13:30:04.725784116Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1298\",\"timestamp\":\"2022-08-12T13:30:04.725808942Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1299\",\"timestamp\":\"2022-08-12T13:30:04.725856476Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1300\",\"timestamp\":\"2022-08-12T13:30:04.725857842Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1301\",\"timestamp\":\"2022-08-12T13:30:04.72586863Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1302\",\"timestamp\":\"2022-08-12T13:30:04.725892592Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1303\",\"timestamp\":\"2022-08-12T13:30:04.725949233Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1304\",\"timestamp\":\"2022-08-12T13:30:04.7259519Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1305\",\"timestamp\":\"2022-08-12T13:30:04.72597082Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1306\",\"timestamp\":\"2022-08-12T13:30:04.726001502Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1307\",\"timestamp\":\"2022-08-12T13:30:04.726068619Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1308\",\"timestamp\":\"2022-08-12T13:30:04.726070651Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1309\",\"timestamp\":\"2022-08-12T13:30:04.7260833Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1310\",\"timestamp\":\"2022-08-12T13:30:04.726108317Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1311\",\"timestamp\":\"2022-08-12T13:30:04.726159542Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1312\",\"timestamp\":\"2022-08-12T13:30:04.726160946Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1313\",\"timestamp\":\"2022-08-12T13:30:04.726174374Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1314\",\"timestamp\":\"2022-08-12T13:30:04.726198185Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1315\",\"timestamp\":\"2022-08-12T13:30:04.726243046Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1316\",\"timestamp\":\"2022-08-12T13:30:04.726244437Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1317\",\"timestamp\":\"2022-08-12T13:30:04.72625513Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1318\",\"timestamp\":\"2022-08-12T13:30:04.726279139Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1319\",\"timestamp\":\"2022-08-12T13:30:04.726328104Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1320\",\"timestamp\":\"2022-08-12T13:30:04.726329551Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1321\",\"timestamp\":\"2022-08-12T13:30:04.726340628Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1322\",\"timestamp\":\"2022-08-12T13:30:04.726363983Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1325\",\"timestamp\":\"2022-08-12T13:30:04.726426695Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1323\",\"timestamp\":\"2022-08-12T13:30:04.726414779Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1326\",\"timestamp\":\"2022-08-12T13:30:04.726450041Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1324\",\"timestamp\":\"2022-08-12T13:30:04.726416144Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1327\",\"timestamp\":\"2022-08-12T13:30:04.726502988Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1328\",\"timestamp\":\"2022-08-12T13:30:04.726504422Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1329\",\"timestamp\":\"2022-08-12T13:30:04.726515286Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1330\",\"timestamp\":\"2022-08-12T13:30:04.726538294Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1331\",\"timestamp\":\"2022-08-12T13:30:04.726589573Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1332\",\"timestamp\":\"2022-08-12T13:30:04.726590938Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1333\",\"timestamp\":\"2022-08-12T13:30:04.726604194Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1334\",\"timestamp\":\"2022-08-12T13:30:04.726629202Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1335\",\"timestamp\":\"2022-08-12T13:30:04.726677631Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1337\",\"timestamp\":\"2022-08-12T13:30:04.726691686Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1336\",\"timestamp\":\"2022-08-12T13:30:04.72667899Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1338\",\"timestamp\":\"2022-08-12T13:30:04.7267156Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1339\",\"timestamp\":\"2022-08-12T13:30:04.726766152Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1340\",\"timestamp\":\"2022-08-12T13:30:04.726767545Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1341\",\"timestamp\":\"2022-08-12T13:30:04.726779868Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1342\",\"timestamp\":\"2022-08-12T13:30:04.726803769Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1343\",\"timestamp\":\"2022-08-12T13:30:04.726866848Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1344\",\"timestamp\":\"2022-08-12T13:30:04.726868706Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1345\",\"timestamp\":\"2022-08-12T13:30:04.72688061Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1346\",\"timestamp\":\"2022-08-12T13:30:04.726914339Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1347\",\"timestamp\":\"2022-08-12T13:30:04.726970228Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1348\",\"timestamp\":\"2022-08-12T13:30:04.726971719Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1349\",\"timestamp\":\"2022-08-12T13:30:04.726986218Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1357\",\"timestamp\":\"2022-08-12T13:30:04.727151561Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1350\",\"timestamp\":\"2022-08-12T13:30:04.727011351Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1358\",\"timestamp\":\"2022-08-12T13:30:04.727185826Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1351\",\"timestamp\":\"2022-08-12T13:30:04.727058523Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1359\",\"timestamp\":\"2022-08-12T13:30:04.727233736Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1352\",\"timestamp\":\"2022-08-12T13:30:04.727059973Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1360\",\"timestamp\":\"2022-08-12T13:30:04.727235228Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1353\",\"timestamp\":\"2022-08-12T13:30:04.727071812Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1354\",\"timestamp\":\"2022-08-12T13:30:04.727096108Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1355\",\"timestamp\":\"2022-08-12T13:30:04.727138971Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1356\",\"timestamp\":\"2022-08-12T13:30:04.727140372Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1361\",\"timestamp\":\"2022-08-12T13:30:04.727247023Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1362\",\"timestamp\":\"2022-08-12T13:30:04.727270416Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1363\",\"timestamp\":\"2022-08-12T13:30:04.727314473Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1364\",\"timestamp\":\"2022-08-12T13:30:04.72731596Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1365\",\"timestamp\":\"2022-08-12T13:30:04.727327822Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1366\",\"timestamp\":\"2022-08-12T13:30:04.727350628Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1369\",\"timestamp\":\"2022-08-12T13:30:04.727410743Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1367\",\"timestamp\":\"2022-08-12T13:30:04.727395569Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1370\",\"timestamp\":\"2022-08-12T13:30:04.727434352Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1368\",\"timestamp\":\"2022-08-12T13:30:04.727397165Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1371\",\"timestamp\":\"2022-08-12T13:30:04.727474503Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1372\",\"timestamp\":\"2022-08-12T13:30:04.727475918Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1373\",\"timestamp\":\"2022-08-12T13:30:04.727488454Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1374\",\"timestamp\":\"2022-08-12T13:30:04.727512885Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1375\",\"timestamp\":\"2022-08-12T13:30:04.727554338Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1376\",\"timestamp\":\"2022-08-12T13:30:04.727555746Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1377\",\"timestamp\":\"2022-08-12T13:30:04.727566931Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1378\",\"timestamp\":\"2022-08-12T13:30:04.727590862Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1379\",\"timestamp\":\"2022-08-12T13:30:04.727651378Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1380\",\"timestamp\":\"2022-08-12T13:30:04.727653445Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1381\",\"timestamp\":\"2022-08-12T13:30:04.72766453Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1382\",\"timestamp\":\"2022-08-12T13:30:04.727693605Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1383\",\"timestamp\":\"2022-08-12T13:30:04.727757231Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1384\",\"timestamp\":\"2022-08-12T13:30:04.727758725Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1385\",\"timestamp\":\"2022-08-12T13:30:04.727768967Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1386\",\"timestamp\":\"2022-08-12T13:30:04.727792182Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1387\",\"timestamp\":\"2022-08-12T13:30:04.727855435Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1388\",\"timestamp\":\"2022-08-12T13:30:04.727856854Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1389\",\"timestamp\":\"2022-08-12T13:30:04.72786886Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1390\",\"timestamp\":\"2022-08-12T13:30:04.7278914Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1391\",\"timestamp\":\"2022-08-12T13:30:04.727956863Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1392\",\"timestamp\":\"2022-08-12T13:30:04.72795815Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1393\",\"timestamp\":\"2022-08-12T13:30:04.727971965Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1394\",\"timestamp\":\"2022-08-12T13:30:04.728004329Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1395\",\"timestamp\":\"2022-08-12T13:30:04.728119497Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1401\",\"timestamp\":\"2022-08-12T13:30:04.728394825Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1396\",\"timestamp\":\"2022-08-12T13:30:04.728121353Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1402\",\"timestamp\":\"2022-08-12T13:30:04.728444298Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1403\",\"timestamp\":\"2022-08-12T13:30:04.728538214Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1397\",\"timestamp\":\"2022-08-12T13:30:04.728134505Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1404\",\"timestamp\":\"2022-08-12T13:30:04.728539818Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1398\",\"timestamp\":\"2022-08-12T13:30:04.728185821Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1399\",\"timestamp\":\"2022-08-12T13:30:04.72837971Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1400\",\"timestamp\":\"2022-08-12T13:30:04.728381238Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1405\",\"timestamp\":\"2022-08-12T13:30:04.72855015Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1406\",\"timestamp\":\"2022-08-12T13:30:04.728581422Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1407\",\"timestamp\":\"2022-08-12T13:30:04.728676286Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1408\",\"timestamp\":\"2022-08-12T13:30:04.72867775Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1409\",\"timestamp\":\"2022-08-12T13:30:04.72869072Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1410\",\"timestamp\":\"2022-08-12T13:30:04.728714949Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1411\",\"timestamp\":\"2022-08-12T13:30:04.728826928Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1412\",\"timestamp\":\"2022-08-12T13:30:04.728828315Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1413\",\"timestamp\":\"2022-08-12T13:30:04.728839654Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1414\",\"timestamp\":\"2022-08-12T13:30:04.728873478Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1415\",\"timestamp\":\"2022-08-12T13:30:04.728953805Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1416\",\"timestamp\":\"2022-08-12T13:30:04.728955354Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1417\",\"timestamp\":\"2022-08-12T13:30:04.728966312Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1418\",\"timestamp\":\"2022-08-12T13:30:04.72900211Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1419\",\"timestamp\":\"2022-08-12T13:30:04.729084668Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1421\",\"timestamp\":\"2022-08-12T13:30:04.729100937Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1420\",\"timestamp\":\"2022-08-12T13:30:04.729086148Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1422\",\"timestamp\":\"2022-08-12T13:30:04.72912988Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1423\",\"timestamp\":\"2022-08-12T13:30:04.729222323Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1424\",\"timestamp\":\"2022-08-12T13:30:04.729223759Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1425\",\"timestamp\":\"2022-08-12T13:30:04.72923462Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1426\",\"timestamp\":\"2022-08-12T13:30:04.729260529Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1427\",\"timestamp\":\"2022-08-12T13:30:04.729324011Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1428\",\"timestamp\":\"2022-08-12T13:30:04.729327645Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1429\",\"timestamp\":\"2022-08-12T13:30:04.729357329Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1430\",\"timestamp\":\"2022-08-12T13:30:04.729400301Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1433\",\"timestamp\":\"2022-08-12T13:30:04.729499517Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1431\",\"timestamp\":\"2022-08-12T13:30:04.729480967Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1434\",\"timestamp\":\"2022-08-12T13:30:04.729542358Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1432\",\"timestamp\":\"2022-08-12T13:30:04.729483589Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1435\",\"timestamp\":\"2022-08-12T13:30:04.729591928Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1436\",\"timestamp\":\"2022-08-12T13:30:04.72959343Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1437\",\"timestamp\":\"2022-08-12T13:30:04.729610379Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1438\",\"timestamp\":\"2022-08-12T13:30:04.729637655Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1441\",\"timestamp\":\"2022-08-12T13:30:04.729739369Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1439\",\"timestamp\":\"2022-08-12T13:30:04.729715463Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1445\",\"timestamp\":\"2022-08-12T13:30:04.729873393Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1440\",\"timestamp\":\"2022-08-12T13:30:04.729716956Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1446\",\"timestamp\":\"2022-08-12T13:30:04.729911713Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1447\",\"timestamp\":\"2022-08-12T13:30:04.730000177Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1448\",\"timestamp\":\"2022-08-12T13:30:04.730002293Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1442\",\"timestamp\":\"2022-08-12T13:30:04.729771376Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1443\",\"timestamp\":\"2022-08-12T13:30:04.729850571Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1444\",\"timestamp\":\"2022-08-12T13:30:04.729853238Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1449\",\"timestamp\":\"2022-08-12T13:30:04.730014324Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1450\",\"timestamp\":\"2022-08-12T13:30:04.730040765Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1451\",\"timestamp\":\"2022-08-12T13:30:04.730101644Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1452\",\"timestamp\":\"2022-08-12T13:30:04.730104439Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1453\",\"timestamp\":\"2022-08-12T13:30:04.730127053Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1454\",\"timestamp\":\"2022-08-12T13:30:04.730164471Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1455\",\"timestamp\":\"2022-08-12T13:30:04.730231814Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1456\",\"timestamp\":\"2022-08-12T13:30:04.730233141Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1461\",\"timestamp\":\"2022-08-12T13:30:04.730397999Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1457\",\"timestamp\":\"2022-08-12T13:30:04.730245355Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1462\",\"timestamp\":\"2022-08-12T13:30:04.730423389Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1458\",\"timestamp\":\"2022-08-12T13:30:04.730280749Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1463\",\"timestamp\":\"2022-08-12T13:30:04.730515122Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1459\",\"timestamp\":\"2022-08-12T13:30:04.730385717Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1464\",\"timestamp\":\"2022-08-12T13:30:04.730516913Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1465\",\"timestamp\":\"2022-08-12T13:30:04.730529857Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1466\",\"timestamp\":\"2022-08-12T13:30:04.730562975Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1467\",\"timestamp\":\"2022-08-12T13:30:04.730634337Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1468\",\"timestamp\":\"2022-08-12T13:30:04.730635776Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1460\",\"timestamp\":\"2022-08-12T13:30:04.730387184Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1469\",\"timestamp\":\"2022-08-12T13:30:04.730648601Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1470\",\"timestamp\":\"2022-08-12T13:30:04.730673802Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1471\",\"timestamp\":\"2022-08-12T13:30:04.730745535Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1472\",\"timestamp\":\"2022-08-12T13:30:04.73074692Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1473\",\"timestamp\":\"2022-08-12T13:30:04.730757894Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1474\",\"timestamp\":\"2022-08-12T13:30:04.730805758Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1475\",\"timestamp\":\"2022-08-12T13:30:04.730872523Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1476\",\"timestamp\":\"2022-08-12T13:30:04.73087409Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1477\",\"timestamp\":\"2022-08-12T13:30:04.730896539Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1478\",\"timestamp\":\"2022-08-12T13:30:04.730943956Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1479\",\"timestamp\":\"2022-08-12T13:30:04.731014997Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1480\",\"timestamp\":\"2022-08-12T13:30:04.731017251Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1481\",\"timestamp\":\"2022-08-12T13:30:04.73107221Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1482\",\"timestamp\":\"2022-08-12T13:30:04.731116021Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1483\",\"timestamp\":\"2022-08-12T13:30:04.731185458Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1484\",\"timestamp\":\"2022-08-12T13:30:04.731187002Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1485\",\"timestamp\":\"2022-08-12T13:30:04.731197755Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1486\",\"timestamp\":\"2022-08-12T13:30:04.731238979Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1493\",\"timestamp\":\"2022-08-12T13:30:04.731428694Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1487\",\"timestamp\":\"2022-08-12T13:30:04.73131753Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1494\",\"timestamp\":\"2022-08-12T13:30:04.731468104Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1488\",\"timestamp\":\"2022-08-12T13:30:04.73131909Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1495\",\"timestamp\":\"2022-08-12T13:30:04.731529004Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1496\",\"timestamp\":\"2022-08-12T13:30:04.731530374Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1489\",\"timestamp\":\"2022-08-12T13:30:04.731329445Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1490\",\"timestamp\":\"2022-08-12T13:30:04.731354836Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1491\",\"timestamp\":\"2022-08-12T13:30:04.731416234Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1492\",\"timestamp\":\"2022-08-12T13:30:04.731417833Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1497\",\"timestamp\":\"2022-08-12T13:30:04.731540925Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1498\",\"timestamp\":\"2022-08-12T13:30:04.731566333Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1499\",\"timestamp\":\"2022-08-12T13:30:04.731622626Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n{\"cluster\":\"infrastage\",\"component\":\"stdout\",\"container_id\":\"d4a92a653f5f\",\"container_name\":\"main\",\"hostname\":\"10-40-22-18-uswest1cdevc\",\"instance\":\"test_output.foo\",\"level\":\"debug\",\"message\":\"1500\",\"timestamp\":\"2022-08-12T13:30:04.731687734Z\",\"pod_name\":\"compute-infra-test-service.test--output.228.foo.jawpvt\",\"tron_run_number\":228}\n" }, { "path": "tests/utils/state_test.py", "content": "from testifycompat import assert_equal\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tron.utils import state\n\n\nclass TestStateMachineSimple(TestCase):\n @setup\n def build_machine(self):\n self.state_green = \"green\"\n self.state_red = \"red\"\n\n self.machine = state.Machine(self.state_red, red=dict(true=\"green\"))\n\n def test_transition_many(self):\n # Stay the same\n assert not self.machine.transition(\"missing\")\n assert_equal(self.machine.state, self.state_red)\n\n # Traffic has arrived\n self.machine.transition(\"true\")\n assert_equal(self.machine.state, self.state_green)\n\n # Still traffic\n self.machine.transition(\"true\")\n assert_equal(self.machine.state, self.state_green)\n\n def test_check(self):\n assert not self.machine.check(False)\n assert_equal(self.machine.check(\"true\"), self.state_green)\n assert_equal(self.machine.state, self.state_red)\n\n\nclass TestStateMachineMultiOption(TestCase):\n @setup\n def build_machine(self):\n # Generalized rules of a conversation\n # If they are talking, we should listen\n # If they are listening, we should talk\n # If they are ignoring us we should get angry\n self.machine = state.Machine(\n \"listening\",\n listening=dict(listening=\"talking\"),\n talking=dict(ignoring=\"angry\", talking=\"listening\"),\n )\n\n def test_transition_many(self):\n # Talking, we should listen\n self.machine.transition(\"talking\")\n assert_equal(self.machine.state, \"listening\")\n\n # Now be polite\n self.machine.transition(\"listening\")\n assert_equal(self.machine.state, \"talking\")\n\n self.machine.transition(\"listening\")\n assert_equal(self.machine.state, \"talking\")\n\n # But they are tired of us...\n self.machine.transition(\"ignoring\")\n assert_equal(self.machine.state, \"angry\")\n\n def test_transition_set(self):\n expected = {\"listening\", \"talking\", \"ignoring\"}\n assert_equal(set(self.machine.transition_names), expected)\n" }, { "path": "tests/utils/timeutils_test.py", "content": "import datetime\n\nimport pytz\n\nfrom testifycompat import assert_equal\nfrom testifycompat import setup\nfrom testifycompat import TestCase\nfrom tests import testingutils\nfrom tron.utils import timeutils\nfrom tron.utils.timeutils import DateArithmetic\nfrom tron.utils.timeutils import duration\nfrom tron.utils.timeutils import macro_timedelta\n\n\nclass TestTimeDelta(TestCase):\n @setup\n def make_dates(self):\n self.start_nonleap = datetime.datetime(year=2011, month=1, day=1)\n self.end_nonleap = datetime.datetime(year=2011, month=12, day=31)\n self.begin_feb_nonleap = datetime.datetime(year=2011, month=2, day=1)\n self.start_leap = datetime.datetime(year=2012, month=1, day=1)\n self.end_leap = datetime.datetime(year=2012, month=12, day=31)\n self.begin_feb_leap = datetime.datetime(year=2012, month=2, day=1)\n\n def check_delta(self, start, target, years=0, months=0, days=0):\n assert_equal(\n start\n + macro_timedelta(\n start,\n years=years,\n months=months,\n days=days,\n ),\n target,\n )\n\n def test_days(self):\n self.check_delta(\n self.start_nonleap,\n datetime.datetime(year=2011, month=1, day=11),\n days=10,\n )\n self.check_delta(\n self.end_nonleap,\n datetime.datetime(year=2012, month=1, day=10),\n days=10,\n )\n self.check_delta(\n self.start_leap,\n datetime.datetime(year=2012, month=1, day=11),\n days=10,\n )\n self.check_delta(\n self.end_leap,\n datetime.datetime(year=2013, month=1, day=10),\n days=10,\n )\n self.check_delta(\n self.begin_feb_nonleap,\n datetime.datetime(year=2011, month=3, day=1),\n days=28,\n )\n self.check_delta(\n self.begin_feb_leap,\n datetime.datetime(year=2012, month=3, day=1),\n days=29,\n )\n\n def test_months(self):\n self.check_delta(\n self.start_nonleap,\n datetime.datetime(year=2011, month=11, day=1),\n months=10,\n )\n self.check_delta(\n self.end_nonleap,\n datetime.datetime(year=2012, month=10, day=31),\n months=10,\n )\n self.check_delta(\n self.start_leap,\n datetime.datetime(year=2012, month=11, day=1),\n months=10,\n )\n self.check_delta(\n self.end_leap,\n datetime.datetime(year=2013, month=10, day=31),\n months=10,\n )\n self.check_delta(\n self.begin_feb_nonleap,\n datetime.datetime(year=2011, month=12, day=1),\n months=10,\n )\n self.check_delta(\n self.begin_feb_leap,\n datetime.datetime(year=2012, month=12, day=1),\n months=10,\n )\n\n def test_years(self):\n self.check_delta(\n self.start_nonleap,\n datetime.datetime(year=2015, month=1, day=1),\n years=4,\n )\n self.check_delta(\n self.end_nonleap,\n datetime.datetime(year=2015, month=12, day=31),\n years=4,\n )\n self.check_delta(\n self.start_leap,\n datetime.datetime(year=2016, month=1, day=1),\n years=4,\n )\n self.check_delta(\n self.end_leap,\n datetime.datetime(year=2016, month=12, day=31),\n years=4,\n )\n self.check_delta(\n self.begin_feb_nonleap,\n datetime.datetime(year=2015, month=2, day=1),\n years=4,\n )\n self.check_delta(\n self.begin_feb_leap,\n datetime.datetime(year=2016, month=2, day=1),\n years=4,\n )\n\n def test_start_date_with_timezone(self):\n pacific_tz = pytz.timezone(\"US/Pacific\")\n start_date = pacific_tz.localize(\n datetime.datetime(year=2018, month=1, day=3, hour=13),\n )\n expected_end = pacific_tz.localize(\n datetime.datetime(year=2018, month=1, day=1, hour=13),\n )\n self.check_delta(\n start_date,\n expected_end,\n days=-2,\n )\n\n\nclass TestDuration(TestCase):\n @setup\n def setup_times(self):\n self.earliest = datetime.datetime(2012, 2, 1, 3, 0, 0)\n self.latest = datetime.datetime(2012, 2, 1, 3, 20, 0)\n\n def test_duration(self):\n assert_equal(\n duration(self.earliest, self.latest),\n datetime.timedelta(0, 60 * 20),\n )\n\n def test_duration_no_end(self):\n delta = duration(self.earliest)\n assert delta.days >= 40\n\n def test_duration_no_start(self):\n assert_equal(duration(None), None)\n\n\nclass TestDeltaTotalSeconds(TestCase):\n def test(self):\n expected = 86702.004002999995\n delta = datetime.timedelta(*range(1, 6))\n delta_seconds = timeutils.delta_total_seconds(delta)\n assert_equal(delta_seconds, expected)\n\n\nclass DateArithmeticTestCase(testingutils.MockTimeTestCase):\n\n # Set a date with days less then 28, otherwise some tests will fail\n # when run on days > 28.\n now = datetime.datetime(2012, 3, 20)\n\n def _cmp_date(self, item, dt):\n assert_equal(DateArithmetic.parse(item), dt.strftime(\"%Y-%m-%d\"))\n\n def _cmp_day(self, item, dt):\n assert_equal(DateArithmetic.parse(item), dt.strftime(\"%d\"))\n\n def _cmp_month(self, item, dt):\n assert_equal(DateArithmetic.parse(item), dt.strftime(\"%m\"))\n\n def _cmp_year(self, item, dt):\n assert_equal(DateArithmetic.parse(item), dt.strftime(\"%Y\"))\n\n def test_shortdate(self):\n self._cmp_date(\"shortdate\", self.now)\n\n def test_shortdate_plus(self):\n for i in range(50):\n dt = self.now + datetime.timedelta(days=i)\n self._cmp_date(\"shortdate+%s\" % i, dt)\n\n def test_shortdate_minus(self):\n for i in range(50):\n dt = self.now - datetime.timedelta(days=i)\n self._cmp_date(\"shortdate-%s\" % i, dt)\n\n def test_day(self):\n self._cmp_day(\"day\", self.now)\n\n def test_day_minus(self):\n for i in range(50):\n dt = self.now - datetime.timedelta(days=i)\n self._cmp_day(\"day-%s\" % i, dt)\n\n def test_day_plus(self):\n for i in range(50):\n dt = self.now + datetime.timedelta(days=i)\n self._cmp_day(\"day+%s\" % i, dt)\n\n def test_month(self):\n self._cmp_month(\"month\", self.now)\n\n def test_month_plus(self):\n for i in range(50):\n dt = self.now + timeutils.macro_timedelta(self.now, months=i)\n self._cmp_month(\"month+%s\" % i, dt)\n\n def test_month_minus(self):\n for i in range(50):\n dt = self.now - timeutils.macro_timedelta(self.now, months=i)\n self._cmp_month(\"month-%s\" % i, dt)\n\n def test_year(self):\n self._cmp_year(\"year\", self.now)\n\n def test_year_plus(self):\n for i in range(50):\n dt = self.now + timeutils.macro_timedelta(self.now, years=i)\n self._cmp_year(\"year+%s\" % i, dt)\n\n def test_year_minus(self):\n for i in range(50):\n dt = self.now - timeutils.macro_timedelta(self.now, years=i)\n self._cmp_year(\"year-%s\" % i, dt)\n\n def test_unixtime(self):\n timestamp = int(self.now.timestamp())\n assert_equal(DateArithmetic.parse(\"unixtime\"), timestamp)\n\n def test_unixtime_plus(self):\n timestamp = int(self.now.timestamp()) + 100\n assert_equal(DateArithmetic.parse(\"unixtime+100\"), timestamp)\n\n def test_unixtime_minus(self):\n timestamp = int(self.now.timestamp()) - 99\n assert_equal(DateArithmetic.parse(\"unixtime-99\"), timestamp)\n\n def test_daynumber(self):\n daynum = self.now.toordinal()\n assert_equal(DateArithmetic.parse(\"daynumber\"), daynum)\n\n def test_daynumber_plus(self):\n daynum = self.now.toordinal() + 1\n assert_equal(DateArithmetic.parse(\"daynumber+1\"), daynum)\n\n def test_daynumber_minus(self):\n daynum = self.now.toordinal() - 1\n assert_equal(DateArithmetic.parse(\"daynumber-1\"), daynum)\n\n def test_hour(self):\n hour = self.now.strftime(\"%H\")\n assert_equal(DateArithmetic.parse(\"hour\"), hour)\n\n def test_hour_plus(self):\n hour = \"%02d\" % ((int(self.now.strftime(\"%H\")) + 1) % 24)\n assert_equal(DateArithmetic.parse(\"hour+1\"), hour)\n\n def test_hour_minus(self):\n hour = \"%02d\" % ((int(self.now.strftime(\"%H\")) - 1) % 24)\n assert_equal(DateArithmetic.parse(\"hour-1\"), hour)\n\n def test_bad_date_format(self):\n assert DateArithmetic.parse(\"~~\") is None\n\n def test_round_day(self):\n start = datetime.datetime(2019, 3, 30)\n delta = timeutils.macro_timedelta(start, months=-1)\n assert (start + delta).day == 28\n\n\nclass DateArithmeticYMDHTest(TestCase):\n def test_ym_plus(self):\n def parse(*ym):\n return DateArithmetic.parse(\"ym+1\", datetime.datetime(*ym))\n\n assert_equal(parse(2018, 1, 1), \"2018-02\")\n assert_equal(parse(2017, 12, 1), \"2018-01\")\n\n def test_ym_minus(self):\n def parse(*ym):\n return DateArithmetic.parse(\"ym-1\", datetime.datetime(*ym))\n\n assert_equal(parse(2018, 1, 1), \"2017-12\")\n assert_equal(parse(2018, 2, 1), \"2018-01\")\n\n def test_ymd_plus(self):\n def parse(*ymd):\n return DateArithmetic.parse(\"ymd+1\", datetime.datetime(*ymd))\n\n assert_equal(parse(2018, 1, 1), \"2018-01-02\")\n assert_equal(parse(2018, 1, 31), \"2018-02-01\")\n\n def test_ymd_minus(self):\n def parse(*ymd):\n return DateArithmetic.parse(\"ymd-1\", datetime.datetime(*ymd))\n\n assert_equal(parse(2018, 1, 1), \"2017-12-31\")\n assert_equal(parse(2018, 1, 2), \"2018-01-01\")\n\n def test_ymdh_plus(self):\n def parse(*ymdh):\n return DateArithmetic.parse(\"ymdh+1\", datetime.datetime(*ymdh))\n\n assert_equal(parse(2018, 1, 1, 1), \"2018-01-01T02\")\n assert_equal(parse(2018, 1, 31, 23), \"2018-02-01T00\")\n\n def test_ymdh_minus(self):\n def parse(*ymdh):\n return DateArithmetic.parse(\"ymdh-1\", datetime.datetime(*ymdh))\n\n assert_equal(parse(2018, 1, 1, 1), \"2018-01-01T00\")\n assert_equal(parse(2018, 1, 1, 0), \"2017-12-31T23\")\n\n def test_ymdhm_plus(self):\n def parse(*ymdhm):\n return DateArithmetic.parse(\"ymdhm+1\", datetime.datetime(*ymdhm))\n\n assert_equal(parse(2018, 1, 1, 1, 1), \"2018-01-01T01:02\")\n assert_equal(parse(2018, 1, 31, 23, 59), \"2018-02-01T00:00\")\n\n def test_ymdhm_minus(self):\n def parse(*ymdhm):\n return DateArithmetic.parse(\"ymdhm-1\", datetime.datetime(*ymdhm))\n\n assert_equal(parse(2018, 1, 1, 1, 2), \"2018-01-01T01:01\")\n assert_equal(parse(2018, 1, 1, 0, 0), \"2017-12-31T23:59\")\n\n def test_ym_minus_round(self):\n dt = datetime.datetime(2019, 3, 30)\n s = timeutils.DateArithmetic.parse(\"ym-1\", dt=dt)\n assert s == \"2019-02\"\n\n def test_ymd_plus_whitespace(self):\n def parse(*ymd):\n return DateArithmetic.parse(\"ymd + 1\", datetime.datetime(*ymd))\n\n assert_equal(parse(2018, 1, 1), \"2018-01-02\")\n assert_equal(parse(2018, 1, 31), \"2018-02-01\")\n\n def test_ymd_minus_whitespace(self):\n def parse(*ymd):\n return DateArithmetic.parse(\"ymd- 1\", datetime.datetime(*ymd))\n\n assert_equal(parse(2018, 1, 1), \"2017-12-31\")\n assert_equal(parse(2018, 1, 2), \"2018-01-01\")\n\n\nclass TestDateArithmeticWithTimezone(DateArithmeticTestCase):\n\n now = pytz.timezone(\"US/Pacific\").localize(datetime.datetime(2012, 3, 20))\n" }, { "path": "tests/utils/trontimespec_test.py", "content": "import datetime\n\nimport pytz\n\nfrom testifycompat import assert_equal\nfrom testifycompat import run\nfrom testifycompat import TestCase\nfrom tron.utils import trontimespec\n\n\nclass TestGetTime(TestCase):\n def test_get_time(self):\n assert_equal(datetime.time(4, 15), trontimespec.get_time(\"4:15\"))\n assert_equal(datetime.time(22, 59), trontimespec.get_time(\"22:59\"))\n\n def test_get_time_invalid_time(self):\n assert not trontimespec.get_time(\"25:00\")\n assert not trontimespec.get_time(\"22:61\")\n\n\nclass TestTimeSpecification(TestCase):\n def _cmp(self, start_time, expected):\n start_time = datetime.datetime(*start_time)\n expected = datetime.datetime(*expected)\n assert_equal(self.time_spec.get_match(start_time), expected)\n\n def test_get_match_months(self):\n self.time_spec = trontimespec.TimeSpecification(months=[1, 5])\n self._cmp((2012, 3, 14), (2012, 5, 1))\n self._cmp((2012, 5, 22), (2012, 5, 23))\n self._cmp((2012, 12, 22), (2013, 1, 1))\n\n def test_get_match_monthdays(self):\n self.time_spec = trontimespec.TimeSpecification(\n monthdays=[10, 3, 3, 10],\n )\n self._cmp((2012, 3, 14), (2012, 4, 3))\n self._cmp((2012, 3, 1), (2012, 3, 3))\n\n def test_get_match_weekdays(self):\n self.time_spec = trontimespec.TimeSpecification(weekdays=[2, 3])\n self._cmp((2012, 3, 14), (2012, 3, 20))\n self._cmp((2012, 3, 20), (2012, 3, 21))\n\n def test_next_month_generator(self):\n time_spec = trontimespec.TimeSpecification(months=[2, 5])\n gen = time_spec.next_month(datetime.datetime(2012, 3, 14))\n expected = [(5, 2012), (2, 2013), (5, 2013), (2, 2014)]\n assert_equal([next(gen) for _ in range(4)], expected)\n\n def test_next_day_monthdays(self):\n time_spec = trontimespec.TimeSpecification(monthdays=[5, 10, 15])\n gen = time_spec.next_day(14, 2012, 3)\n assert_equal(list(gen), [15])\n\n gen = time_spec.next_day(1, 2012, 3)\n assert_equal(list(gen), [5, 10, 15])\n\n def test_next_day_monthdays_with_last(self):\n time_spec = trontimespec.TimeSpecification(monthdays=[5, \"LAST\"])\n gen = time_spec.next_day(14, 2012, 3)\n assert_equal(list(gen), [31])\n\n def test_next_day_weekdays(self):\n time_spec = trontimespec.TimeSpecification(weekdays=[1, 5])\n gen = time_spec.next_day(14, 2012, 3)\n assert_equal(list(gen), [16, 19, 23, 26, 30])\n\n gen = time_spec.next_day(1, 2012, 3)\n assert_equal(list(gen), [2, 5, 9, 12, 16, 19, 23, 26, 30])\n\n def test_next_day_weekdays_with_ordinals(self):\n time_spec = trontimespec.TimeSpecification(\n weekdays=[1, 5],\n ordinals=[1, 3],\n )\n gen = time_spec.next_day(14, 2012, 3)\n assert_equal(list(gen), [16, 19])\n\n gen = time_spec.next_day(1, 2012, 3)\n assert_equal(list(gen), [2, 5, 16, 19])\n\n def test_next_time_timestr(self):\n time_spec = trontimespec.TimeSpecification(timestr=\"13:13\")\n start_date = datetime.datetime(2012, 3, 14, 0, 15)\n time = time_spec.next_time(start_date, True)\n assert_equal(time, datetime.time(13, 13))\n\n start_date = datetime.datetime(2012, 3, 14, 13, 13)\n assert time_spec.next_time(start_date, True) is None\n time = time_spec.next_time(start_date, False)\n assert_equal(time, datetime.time(13, 13))\n\n def test_next_time_hours(self):\n time_spec = trontimespec.TimeSpecification(hours=[4, 10])\n start_date = datetime.datetime(2012, 3, 14, 0, 15)\n time = time_spec.next_time(start_date, True)\n assert_equal(time, datetime.time(4, 0))\n\n start_date = datetime.datetime(2012, 3, 14, 13, 13)\n assert time_spec.next_time(start_date, True) is None\n time = time_spec.next_time(start_date, False)\n assert_equal(time, datetime.time(4, 0))\n\n def test_next_time_minutes(self):\n time_spec = trontimespec.TimeSpecification(\n minutes=[30, 20, 30],\n seconds=[0],\n )\n start_date = datetime.datetime(2012, 3, 14, 0, 25)\n time = time_spec.next_time(start_date, True)\n assert_equal(time, datetime.time(0, 30))\n\n start_date = datetime.datetime(2012, 3, 14, 23, 30)\n assert time_spec.next_time(start_date, True) is None\n time = time_spec.next_time(start_date, False)\n assert_equal(time, datetime.time(0, 20))\n\n def test_next_time_hours_and_minutes_and_seconds(self):\n time_spec = trontimespec.TimeSpecification(\n minutes=[20, 30],\n hours=[1, 5],\n seconds=[4, 5],\n )\n start_date = datetime.datetime(2012, 3, 14, 1, 25)\n time = time_spec.next_time(start_date, True)\n assert_equal(time, datetime.time(1, 30, 4))\n\n start_date = datetime.datetime(2012, 3, 14, 5, 30, 6)\n assert time_spec.next_time(start_date, True) is None\n time = time_spec.next_time(start_date, False)\n assert_equal(time, datetime.time(1, 20, 4))\n\n def test_get_match_dst_spring_forward(self):\n tz = pytz.timezone(\"US/Pacific\")\n time_spec = trontimespec.TimeSpecification(\n hours=[0, 1, 2, 3, 4],\n minutes=[0],\n seconds=[0],\n timezone=\"US/Pacific\",\n )\n start = trontimespec.naive_as_timezone(datetime.datetime(2020, 3, 8, 1), tz)\n # Springing forward, the next hour after 1AM should be 3AM\n next_time = time_spec.get_match(start)\n assert next_time.hour == 3\n\n def test_get_match_dst_fall_back(self):\n tz = pytz.timezone(\"US/Pacific\")\n time_spec = trontimespec.TimeSpecification(\n hours=[0, 1, 2, 3, 4],\n minutes=[0],\n seconds=[0],\n timezone=\"US/Pacific\",\n )\n start = trontimespec.naive_as_timezone(datetime.datetime(2020, 11, 1, 1), tz)\n # Falling back, the next hour after 1AM is 1AM again. But we only run on the first 1AM\n # Next run time should be 2AM\n next_time = time_spec.get_match(start)\n assert next_time.hour == 2\n\n\nif __name__ == \"__main__\":\n run()\n" }, { "path": "tools/action_dag_diagram.py", "content": "\"\"\"\n Create a graphviz diagram from a Tron Job configuration.\n\n Usage:\n python tools/action_dag_diagram.py -c -n \n\n This will create a file named .dot\n You can create a diagram using:\n dot -Tpng -o .png .dot\n\"\"\"\nimport optparse\n\nfrom tron.config import manager\nfrom tron.config import schema\n\n\ndef parse_args():\n parser = optparse.OptionParser()\n parser.add_option(\"-c\", \"--config\", help=\"Tron configuration path.\")\n parser.add_option(\n \"-n\",\n \"--name\",\n help=\"Job name to graph. Also used as output filename.\",\n )\n parser.add_option(\n \"--namespace\",\n default=schema.MASTER_NAMESPACE,\n help=\"Configuration namespace which contains the job.\",\n )\n opts, _ = parser.parse_args()\n\n if not opts.config:\n parser.error(\"A config filename is required.\")\n if not opts.name:\n parser.error(\"A Job name is required.\")\n return opts\n\n\ndef build_diagram(job_config):\n edges, nodes = [], []\n\n for action in job_config.actions.values():\n shape = \"invhouse\" if not action.requires else \"rect\"\n nodes.append(f\"node [shape = {shape}]; {action.name}\")\n for required_action in action.requires:\n edges.append(f\"{required_action} -> {action.name}\")\n\n return \"digraph g{{{}\\n{}}}\".format(\"\\n\".join(nodes), \"\\n\".join(edges))\n\n\ndef get_job(config_container, namespace, job_name):\n if namespace not in config_container:\n raise ValueError(\"Unknown namespace: %s\" % namespace)\n\n config = config_container[opts.namespace]\n if job_name not in config.jobs:\n raise ValueError(\"Could not find Job %s\" % job_name)\n\n return config.jobs[job_name]\n\n\nif __name__ == \"__main__\":\n opts = parse_args()\n\n config_manager = manager.ConfigManager(opts.config)\n container = config_manager.load()\n job_config = get_job(container, opts.namespace, opts.name)\n graph = build_diagram(job_config)\n\n with open(\"%s.dot\" % opts.name, \"w\") as fh:\n fh.write(graph)\n" }, { "path": "tools/compress_json.py", "content": "import argparse\nimport gzip\nimport math\nimport os\nimport sys\nimport threading\nimport time\nfrom concurrent.futures import as_completed\nfrom concurrent.futures import ThreadPoolExecutor\n\nimport boto3\nfrom boto3.resources.base import ServiceResource\n\nfrom tron.core.job import Job\nfrom tron.core.jobrun import JobRun\nfrom tron.serialize import runstate\n\n# Max DynamoDB object size is 400KB. Since we save two copies of the object (pickled and JSON),\n# we need to consider this max size applies to the entire item, so we use a max size of 200KB\n# for each version.\nOBJECT_SIZE = 150_000\n\n# DynamoDB TransactWriteItems supports up to 100 items per call. We use 25\n# which comfortably fits under the 4MB request size limit (25 × 150KB = 3.75MB\n# worst case). This is sufficient to handle our largest items (~20 partitions\n# uncompressed) in a single atomic transaction after compression.\nMAX_TRANSACT_WRITE_ITEMS = 25\n\n\ndef get_dynamodb_table(\n aws_profile: str | None = None, table: str = \"infrastage-tron-state\", region: str = \"us-west-1\"\n) -> ServiceResource:\n session = boto3.Session(profile_name=aws_profile) if aws_profile else boto3.Session()\n return session.resource(\"dynamodb\", region_name=region).Table(table)\n\n\ndef get_dynamodb_client(\n aws_profile: str | None = None,\n region: str = \"us-west-1\",\n):\n session = boto3.Session(profile_name=aws_profile) if aws_profile else boto3.Session()\n return session.client(\"dynamodb\", region_name=region)\n\n\ndef scan_keys(source_table: ServiceResource) -> set[str]:\n \"\"\"Streaming scan that only projects the key attribute and collects unique partition keys.\n\n Never stores full items in memory — only the 'key' strings.\n \"\"\"\n unique_keys: set[str] = set()\n scan_kwargs = {\n \"ProjectionExpression\": \"#k\",\n \"ExpressionAttributeNames\": {\"#k\": \"key\"},\n }\n response = source_table.scan(**scan_kwargs)\n for item in response.get(\"Items\", []):\n unique_keys.add(item.get(\"key\", \"Unknown Key\"))\n while \"LastEvaluatedKey\" in response:\n response = source_table.scan(ExclusiveStartKey=response[\"LastEvaluatedKey\"], **scan_kwargs)\n for item in response.get(\"Items\", []):\n unique_keys.add(item.get(\"key\", \"Unknown Key\"))\n return unique_keys\n\n\ndef resolve_keys(args, parser, source_table: ServiceResource) -> list[str]:\n if not args.keys and not args.keys_file and not args.all:\n parser.error(\"You must provide either --keys, --keys-file, or --all.\")\n\n if args.all:\n print(\"Scanning table for all keys (keys-only projection)...\")\n keys_set = scan_keys(source_table)\n print(f\"Found {len(keys_set)} unique keys.\")\n return list(keys_set)\n\n keys = []\n if args.keys:\n keys.extend(args.keys)\n if args.keys_file:\n try:\n with open(args.keys_file) as f:\n keys_from_file = [line.strip() for line in f if line.strip()]\n keys.extend(keys_from_file)\n except Exception as e:\n parser.error(f\"Error reading keys from file {args.keys_file}: {e}\")\n if not keys:\n parser.error(\"No keys provided. Please provide keys via --keys or --keys-file.\")\n return list(set(keys))\n\n\ndef is_compressed(json_val) -> bool:\n \"\"\"Check if a json_val from DynamoDB is compressed (Binary type) vs uncompressed (String type).\n\n boto3 high-level resource API returns:\n - \"S\" type as Python str\n - \"B\" type as boto3.dynamodb.types.Binary (which wraps bytes and has a .value attribute)\n \"\"\"\n if hasattr(json_val, \"value\"):\n # boto3.dynamodb.types.Binary\n return True\n if isinstance(json_val, bytes):\n # I don't think this is possible in the high-level API, but it's harmless to check\n return True\n return False\n\n\ndef get_json_val_bytes(json_val) -> bytes:\n \"\"\"Extract raw bytes from a json_val, whether it's a Binary wrapper or raw bytes.\"\"\"\n if hasattr(json_val, \"value\"):\n return bytes(json_val.value)\n if isinstance(json_val, bytes):\n return bytes(json_val)\n raise TypeError(f\"Unexpected json_val type: {type(json_val)}\")\n\n\ndef classify_item(item: dict) -> str:\n \"\"\"Classify a partition-0 item into one of: compressed, uncompressed, pickle_only, no_data.\"\"\"\n has_json = \"json_val\" in item\n has_pickle = \"val\" in item\n\n if has_json:\n if is_compressed(item[\"json_val\"]):\n return \"compressed\"\n else:\n return \"uncompressed\"\n elif has_pickle:\n # Shouldn't exist. If we have items with only pickle data we want to know\n return \"pickle_only\"\n else:\n # Shouldn't exist. If we have funky items with no json_val/val we want to know\n return \"no_data\"\n\n\ndef compress_json_for_key(\n source_table: ServiceResource, client, table_name: str, key: str, dry_run: bool = True\n) -> str:\n \"\"\"Compress uncompressed JSON for a single key.\n\n Reads all json_val partitions via get_item (ConsistentRead), gzip-compresses the combined JSON,\n and writes the compressed data back using TransactWriteItems with ConditionExpressions to guard\n against concurrent trond writes.\n\n Returns a status string: \"compressed\", \"already_compressed\", \"no_json\", \"skipped\",\n \"concurrent_update\", or raises on error (including throttle exhaustion).\n \"\"\"\n response = source_table.get_item(Key={\"key\": key, \"index\": 0}, ConsistentRead=True)\n if \"Item\" not in response:\n print(f\" SKIP (no item found): {key}\")\n return \"skipped\"\n\n item_0 = response[\"Item\"]\n\n if \"json_val\" not in item_0:\n print(f\" SKIP (no json_val — needs pickles_to_json.py first): {key}\")\n return \"no_json\"\n\n json_val = item_0[\"json_val\"]\n if is_compressed(json_val):\n print(f\" SKIP (already compressed): {key}\")\n return \"already_compressed\"\n\n # It's an uncompressed string — collect all partitions via get_item\n num_json_partitions = int(item_0.get(\"num_json_val_partitions\", 1))\n combined_json = \"\"\n for index in range(num_json_partitions):\n if index == 0:\n partition_item = item_0\n else:\n resp = source_table.get_item(Key={\"key\": key, \"index\": index}, ConsistentRead=True)\n if \"Item\" not in resp:\n raise Exception(f\"Missing JSON partition {index} for key {key}\")\n partition_item = resp[\"Item\"]\n\n if \"json_val\" not in partition_item:\n raise Exception(f\"No 'json_val' in partition {index} for key {key}\")\n combined_json += partition_item[\"json_val\"]\n\n # Validate JSON round-trips before compressing\n state_type = key.split()[0]\n if state_type == runstate.JOB_STATE:\n Job.from_json(combined_json)\n elif state_type == runstate.JOB_RUN_STATE:\n JobRun.from_json(combined_json)\n else:\n print(f\" SKIP (unknown state type '{state_type}'): {key}\")\n return \"skipped\"\n\n # Compress\n compressed = gzip.compress(combined_json.encode(\"utf-8\"))\n num_compressed_partitions = math.ceil(len(compressed) / OBJECT_SIZE)\n\n original_size = len(combined_json.encode(\"utf-8\"))\n compressed_size = len(compressed)\n ratio = (1 - compressed_size / original_size) * 100 if original_size > 0 else 0\n\n if dry_run:\n print(\n f\" DRY RUN (would compress): {key} \"\n f\"({original_size:,} bytes -> {compressed_size:,} bytes, {ratio:.1f}% reduction, \"\n f\"{num_json_partitions} partitions -> {num_compressed_partitions} partitions)\"\n )\n else:\n # Build TransactWriteItems with conditional expressions.\n # Conditions on partition 0 ensure the item is still uncompressed and has the same\n # number of partitions we read — if trond wrote concurrently the condition fails.\n transact_items = []\n for i in range(num_compressed_partitions):\n chunk = compressed[i * OBJECT_SIZE : (i + 1) * OBJECT_SIZE]\n update = {\n \"Update\": {\n \"TableName\": table_name,\n \"Key\": {\n \"key\": {\"S\": key},\n \"index\": {\"N\": str(i)},\n },\n \"UpdateExpression\": \"SET json_val = :json, num_json_val_partitions = :n\",\n \"ExpressionAttributeValues\": {\n \":json\": {\"B\": chunk},\n \":n\": {\"N\": str(num_compressed_partitions)},\n },\n },\n }\n if i == 0:\n # Condition on partition 0: item must still be uncompressed with expected partition count\n update[\"Update\"][\"ConditionExpression\"] = (\n \"attribute_exists(json_val) \"\n \"AND attribute_type(json_val, :string_type) \"\n \"AND num_json_val_partitions = :expected_partitions\"\n )\n update[\"Update\"][\"ExpressionAttributeValues\"][\":string_type\"] = {\"S\": \"S\"}\n update[\"Update\"][\"ExpressionAttributeValues\"][\":expected_partitions\"] = {\"N\": str(num_json_partitions)}\n transact_items.append(update)\n\n # Clean up excess partitions (REMOVE json_val where compressed needs fewer partitions)\n for i in range(num_compressed_partitions, num_json_partitions):\n transact_items.append(\n {\n \"Update\": {\n \"TableName\": table_name,\n \"Key\": {\n \"key\": {\"S\": key},\n \"index\": {\"N\": str(i)},\n },\n \"UpdateExpression\": \"REMOVE json_val\",\n },\n }\n )\n\n if len(transact_items) > MAX_TRANSACT_WRITE_ITEMS:\n raise Exception(\n f\"Compression requires {len(transact_items)} transaction items for key {key}, \"\n f\"exceeding single-transaction limit of {MAX_TRANSACT_WRITE_ITEMS}. \"\n f\"({num_json_partitions} uncompressed partitions, \"\n f\"{num_compressed_partitions} compressed partitions)\"\n )\n if compressed_size > 3_500_000:\n raise Exception(\n f\"Compressed data too large ({compressed_size:,} bytes) for key {key} — \"\n f\"risks exceeding 4MB TransactWriteItems request limit.\"\n )\n\n # Single atomic transaction — all updates and removes in one call. We retry\n # on ThrottlingException with exponential backoff on top of boto3's built-in\n # retries. During testing we are getting throttled almost immediately (on hot\n # keys) and quickly exhausting the default retry budget. This slows things\n # down a lot but allows the process to complete without manual intervention\n # or re-runs.\n max_retries = 5\n for attempt in range(max_retries + 1):\n try:\n client.transact_write_items(TransactItems=transact_items)\n break\n except client.exceptions.TransactionCanceledException as e:\n reasons = e.response.get(\"CancellationReasons\", [])\n if any(r.get(\"Code\") in (\"ConditionalCheckFailed\", \"TransactionConflict\") for r in reasons):\n print(f\" SKIPPED (concurrent update detected): {key}\")\n return \"concurrent_update\"\n # Check if cancellation was due to throttling\n if any(r.get(\"Code\") == \"ThrottlingError\" for r in reasons):\n if attempt < max_retries:\n wait = 2**attempt\n print(f\" THROTTLED (attempt {attempt + 1}/{max_retries + 1}, retrying in {wait}s): {key}\")\n time.sleep(wait)\n continue\n raise\n except Exception as e:\n if \"ThrottlingException\" in str(e) or \"Throughput exceeds\" in str(e):\n if attempt < max_retries:\n wait = 2**attempt\n print(f\" THROTTLED (attempt {attempt + 1}/{max_retries + 1}, retrying in {wait}s): {key}\")\n time.sleep(wait)\n continue\n raise\n else:\n raise Exception(f\"Throttled after {max_retries + 1} attempts\")\n\n print(\n f\" COMPRESSED: {key} \"\n f\"({original_size:,} bytes -> {compressed_size:,} bytes, {ratio:.1f}% reduction, \"\n f\"{num_json_partitions} partitions -> {num_compressed_partitions} partitions)\"\n )\n\n return \"compressed\"\n\n\ndef verify_compressed_json_for_key(source_table: ServiceResource, key: str) -> bool:\n \"\"\"Re-read all compressed JSON partitions, gunzip, and validate via from_json.\n\n Returns True if verification succeeds, False otherwise. Prints the reason on failure.\n \"\"\"\n response = source_table.get_item(Key={\"key\": key, \"index\": 0}, ConsistentRead=True)\n if \"Item\" not in response:\n print(f\" VERIFY FAIL (no item found): {key}\")\n return False\n\n item_0 = response[\"Item\"]\n\n if \"json_val\" not in item_0:\n print(f\" VERIFY FAIL (no json_val): {key}\")\n return False\n\n if not is_compressed(item_0[\"json_val\"]):\n print(f\" VERIFY FAIL (json_val is not compressed): {key}\")\n return False\n\n num_json_partitions = int(item_0.get(\"num_json_val_partitions\", 1))\n\n # Reassemble compressed bytes from all partitions\n compressed_data = bytearray()\n for index in range(num_json_partitions):\n if index == 0:\n partition_item = item_0\n else:\n resp = source_table.get_item(Key={\"key\": key, \"index\": index}, ConsistentRead=True)\n if \"Item\" not in resp:\n print(f\" VERIFY FAIL (missing partition {index}): {key}\")\n return False\n partition_item = resp[\"Item\"]\n\n if \"json_val\" not in partition_item:\n print(f\" VERIFY FAIL (no json_val in partition {index}): {key}\")\n return False\n\n compressed_data += get_json_val_bytes(partition_item[\"json_val\"])\n\n # Decompress\n try:\n json_str = gzip.decompress(bytes(compressed_data)).decode(\"utf-8\")\n except Exception as e:\n print(f\" VERIFY FAIL (gunzip failed: {e}): {key}\")\n return False\n\n # Validate via from_json\n state_type = key.split()[0]\n try:\n if state_type == runstate.JOB_STATE:\n Job.from_json(json_str)\n elif state_type == runstate.JOB_RUN_STATE:\n JobRun.from_json(json_str)\n else:\n print(f\" VERIFY FAIL (unknown state type '{state_type}'): {key}\")\n return False\n except Exception as e:\n print(f\" VERIFY FAIL (from_json failed: {e}): {key}\")\n return False\n\n return True\n\n\ndef delete_pickle_for_key(source_table: ServiceResource, key: str, dry_run: bool = True) -> str:\n \"\"\"Delete pickle data (val, num_partitions) for a single key.\n\n Verifies compressed JSON can be fully decoded and parsed before deleting.\n\n Returns a status string: \"deleted\", \"refused\", \"no_pickle\", \"skipped\", or raises on error.\n \"\"\"\n response = source_table.get_item(Key={\"key\": key, \"index\": 0}, ConsistentRead=True)\n if \"Item\" not in response:\n print(f\" SKIP (no item found): {key}\")\n return \"skipped\"\n\n item_0 = response[\"Item\"]\n\n # Safety checks\n if \"json_val\" not in item_0:\n print(f\" REFUSE (no json_val at all): {key}\")\n return \"refused\"\n\n if not is_compressed(item_0[\"json_val\"]):\n print(f\" REFUSE (json_val is uncompressed — run 'compress' first): {key}\")\n return \"refused\"\n\n if \"val\" not in item_0:\n print(f\" SKIP (no pickle data to delete): {key}\")\n return \"no_pickle\"\n\n # Verify compressed JSON is valid before deleting pickle\n if not verify_compressed_json_for_key(source_table, key):\n print(f\" REFUSE (compressed JSON verification failed): {key}\")\n return \"refused\"\n\n num_partitions = int(item_0.get(\"num_partitions\", 1))\n num_json_partitions = int(item_0.get(\"num_json_val_partitions\", 1))\n max_partitions = max(num_partitions, num_json_partitions)\n\n if dry_run:\n print(\n f\" DRY RUN (would delete pickle): {key} ({num_partitions} pickle partitions across {max_partitions} items)\"\n )\n else:\n for i in range(max_partitions):\n source_table.update_item(\n Key={\"key\": key, \"index\": i},\n UpdateExpression=\"REMOVE val, num_partitions\",\n )\n print(f\" DELETED pickle: {key} ({num_partitions} pickle partitions removed across {max_partitions} items)\")\n\n return \"deleted\"\n\n\ndef cmd_compress(args, source_table: ServiceResource, client, table_name: str, keys: list[str]) -> None:\n dry_run = not args.execute\n workers = args.workers\n total = len(keys)\n counts = {\n \"compressed\": 0,\n \"already_compressed\": 0,\n \"no_json\": 0,\n \"skipped\": 0,\n \"concurrent_update\": 0,\n \"failed\": 0,\n }\n failed_keys = []\n lock = threading.Lock()\n completed = [0] # mutable counter for progress\n\n mode = \"DRY RUN\" if dry_run else \"EXECUTING\"\n print(f\"\\n=== Compress JSON ({mode}, {workers} workers) ===\")\n print(f\"Processing {total} keys...\\n\")\n\n # Pre-create one Table resource per worker thread. The high-level\n # resource is not thread-safe, so each worker gets its own, but we\n # create them once upfront instead of per-key.\n thread_tables = [get_dynamodb_table(args.aws_profile, args.table_name, args.table_region) for _ in range(workers)]\n # Map thread IDs to table resources as workers claim them.\n thread_local = threading.local()\n\n def get_thread_table() -> ServiceResource:\n if not hasattr(thread_local, \"table\"):\n with lock:\n thread_local.table = thread_tables.pop()\n return thread_local.table\n\n def process_key(key: str) -> None:\n thread_table = get_thread_table()\n try:\n result = compress_json_for_key(thread_table, client, table_name, key, dry_run=dry_run)\n except Exception as e:\n result = \"failed\"\n with lock:\n failed_keys.append(key)\n print(f\" FAILED ({key}): {e}\")\n\n with lock:\n counts[result] += 1\n completed[0] += 1\n if completed[0] % 500 == 0 or completed[0] == total:\n print(f\" Progress: {completed[0]}/{total} keys processed\")\n\n sorted_keys = sorted(keys)\n with ThreadPoolExecutor(max_workers=workers) as pool:\n futures = {pool.submit(process_key, key): key for key in sorted_keys}\n for future in as_completed(futures):\n # Exceptions are already handled inside process_key, but catch\n # anything truly unexpected so one bad future doesn't kill the pool.\n try:\n future.result()\n except Exception as e:\n key = futures[future]\n print(f\" UNEXPECTED ERROR ({key}): {e}\")\n with lock:\n counts[\"failed\"] += 1\n failed_keys.append(key)\n\n print(\"\\n=== Summary ===\")\n print(f\"Total keys: {total}\")\n print(f\"Compressed: {counts['compressed']}\")\n print(f\"Already compressed: {counts['already_compressed']}\")\n print(f\"No JSON (pickle-only):{counts['no_json']}\")\n print(f\"Skipped: {counts['skipped']}\")\n print(f\"Concurrent updates: {counts['concurrent_update']}\")\n print(f\"Failed: {counts['failed']}\")\n\n if dry_run:\n print(\"\\nDry run complete. No changes were made.\")\n\n if args.failed_keys_output and failed_keys:\n with open(args.failed_keys_output, \"w\") as f:\n for key in failed_keys:\n f.write(f\"{key}\\n\")\n print(f\"Failed keys written to {args.failed_keys_output}\")\n\n\ndef cmd_delete_pickles(args, source_table: ServiceResource, keys: list[str]) -> None:\n dry_run = not args.execute\n\n if args.execute and not args.i_hereby_declare_we_no_longer_need_pickles:\n print(\n \"ERROR: --execute for delete-pickles requires the safety flag:\\n\"\n \" --i-hereby-declare-we-no-longer-need-pickles\\n\\n\"\n \"This operation is DESTRUCTIVE and IRREVERSIBLE. It removes all pickle data\\n\"\n \"from DynamoDB items. Only proceed if you are certain that:\\n\"\n \" 1. All items have valid compressed JSON (run 'status' to verify)\\n\"\n \" 2. Tron is configured to read from JSON (read_json=True)\\n\"\n \" 3. You have verified restores work from JSON on replica tables\"\n )\n sys.exit(1)\n\n total = len(keys)\n counts = {\"deleted\": 0, \"refused\": 0, \"no_pickle\": 0, \"skipped\": 0, \"failed\": 0}\n failed_keys = []\n\n mode = \"DRY RUN\" if dry_run else \"EXECUTING\"\n print(f\"\\n=== Delete Pickles ({mode}) ===\")\n print(f\"Processing {total} keys...\\n\")\n\n for i, key in enumerate(sorted(keys), 1):\n print(f\"[{i}/{total}] {key}\")\n try:\n result = delete_pickle_for_key(source_table, key, dry_run=dry_run)\n counts[result] += 1\n except Exception as e:\n print(f\" FAILED: {e}\")\n counts[\"failed\"] += 1\n failed_keys.append(key)\n\n print(\"\\n=== Summary ===\")\n print(f\"Total keys: {total}\")\n print(f\"Deleted: {counts['deleted']}\")\n print(f\"Refused (no comp. JSON): {counts['refused']}\")\n print(f\"No pickle to delete: {counts['no_pickle']}\")\n print(f\"Skipped: {counts['skipped']}\")\n print(f\"Failed: {counts['failed']}\")\n\n if dry_run:\n print(\"\\nDry run complete. No changes were made.\")\n\n if args.failed_keys_output and failed_keys:\n with open(args.failed_keys_output, \"w\") as f:\n for key in failed_keys:\n f.write(f\"{key}\\n\")\n print(f\"Failed keys written to {args.failed_keys_output}\")\n\n\ndef cmd_status(args, source_table: ServiceResource) -> None:\n print(f\"\\n=== Status: {args.table_name} ({args.table_region}) ===\")\n print(\"Scanning partition-0 items...\")\n\n counts = {\"compressed\": 0, \"uncompressed\": 0, \"pickle_only\": 0, \"no_data\": 0}\n total = 0\n\n # Streaming scan filtered to partition 0 only, projecting just the attributes\n # needed for classification. This avoids per-key get_item calls.\n scan_kwargs = {\n \"FilterExpression\": \"#idx = :zero\",\n \"ProjectionExpression\": \"#k, #idx, json_val, val\",\n \"ExpressionAttributeNames\": {\"#k\": \"key\", \"#idx\": \"index\"},\n \"ExpressionAttributeValues\": {\":zero\": 0},\n }\n response = source_table.scan(**scan_kwargs)\n for item in response.get(\"Items\", []):\n total += 1\n counts[classify_item(item)] += 1\n while \"LastEvaluatedKey\" in response:\n response = source_table.scan(ExclusiveStartKey=response[\"LastEvaluatedKey\"], **scan_kwargs)\n for item in response.get(\"Items\", []):\n total += 1\n counts[classify_item(item)] += 1\n\n print(f\"\\nTotal unique keys: {total:,}\\n\")\n for label, count_key in [\n (\"Compressed JSON (ready for pickle deletion)\", \"compressed\"),\n (\"Uncompressed JSON (needs compression)\", \"uncompressed\"),\n (\"Pickle only (anomalous, needs pickles_to_json.py)\", \"pickle_only\"),\n (\"No data (anomalous, needs investigation)\", \"no_data\"),\n ]:\n count = counts[count_key]\n pct = (count / total * 100) if total > 0 else 0\n print(f\" {label + ':':<50} {count:>8,} ({pct:.1f}%)\")\n\n\ndef add_key_arguments(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"--keys\",\n nargs=\"+\",\n required=False,\n help=\"Specific key(s) to process.\",\n )\n subparser.add_argument(\n \"--keys-file\",\n required=False,\n help=\"Input file containing keys to process. One key per line.\",\n )\n subparser.add_argument(\n \"--all\",\n action=\"store_true\",\n help=\"Process all keys in the table.\",\n )\n\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"Compress JSON and delete pickle data in Tron's DynamoDB state store.\",\n epilog=\"\"\"\nSub-commands:\n compress Compress uncompressed JSON (\"S\" type) to gzip-compressed binary (\"B\" type).\n delete-pickles Remove pickle data (val, num_partitions) from items that have compressed JSON.\n status Report the state of all keys in the table.\n\nExamples:\n Check status of a table:\n compress_json.py --table-name infrastage-tron-state --table-region us-west-1 status\n\n Dry-run compression for all keys:\n compress_json.py --table-name infrastage-tron-state --table-region us-west-1 compress --all\n\n Execute compression for specific keys:\n compress_json.py --table-name infrastage-tron-state --table-region us-west-1 compress --keys \"job_state myjob\" --execute\n\n Dry-run pickle deletion:\n compress_json.py --table-name infrastage-tron-state --table-region us-west-1 delete-pickles --all\n\n Execute pickle deletion (requires safety flag):\n compress_json.py --table-name infrastage-tron-state --table-region us-west-1 delete-pickles --all --execute --i-hereby-declare-we-no-longer-need-pickles\n\"\"\",\n formatter_class=argparse.RawDescriptionHelpFormatter,\n )\n parser.add_argument(\n \"--aws-profile\",\n default=os.environ.get(\"AWS_PROFILE\", None),\n help=\"AWS profile to use (default: taken from AWS_PROFILE environment variable)\",\n )\n parser.add_argument(\"--table-name\", required=True, help=\"Name of the DynamoDB table\")\n parser.add_argument(\"--table-region\", required=True, help=\"AWS region of the DynamoDB table\")\n\n subparsers = parser.add_subparsers(dest=\"action\", required=True, help=\"Action to perform\")\n\n # compress sub-command\n compress_parser = subparsers.add_parser(\n \"compress\",\n help=\"Compress uncompressed JSON to gzip-compressed binary.\",\n )\n add_key_arguments(compress_parser)\n compress_parser.add_argument(\n \"--execute\",\n action=\"store_true\",\n default=False,\n help=\"Actually perform the compression. Dry-run by default.\",\n )\n compress_parser.add_argument(\n \"--workers\",\n type=int,\n default=8,\n help=\"Number of concurrent workers (default: 8). Increase for faster throughput; decrease if throttling is excessive.\",\n )\n compress_parser.add_argument(\n \"--failed-keys-output\",\n required=False,\n help=\"Output file to write keys that failed compression. One key per line.\",\n )\n\n # delete-pickles sub-command\n delete_parser = subparsers.add_parser(\n \"delete-pickles\",\n help=\"Remove pickle data from items that have compressed JSON.\",\n )\n add_key_arguments(delete_parser)\n delete_parser.add_argument(\n \"--execute\",\n action=\"store_true\",\n default=False,\n help=\"Actually perform the deletion. Dry-run by default.\",\n )\n delete_parser.add_argument(\n \"--i-hereby-declare-we-no-longer-need-pickles\",\n action=\"store_true\",\n default=False,\n help=\"Required safety flag when using --execute. Confirms you understand this is destructive and irreversible.\",\n )\n delete_parser.add_argument(\n \"--failed-keys-output\",\n required=False,\n help=\"Output file to write keys that failed deletion. One key per line.\",\n )\n\n # status sub-command\n subparsers.add_parser(\n \"status\",\n help=\"Report the state of all keys in the table.\",\n )\n\n args = parser.parse_args()\n source_table = get_dynamodb_table(args.aws_profile, args.table_name, args.table_region)\n client = get_dynamodb_client(args.aws_profile, args.table_region)\n\n if args.action == \"status\":\n cmd_status(args, source_table)\n elif args.action == \"compress\":\n keys = resolve_keys(args, parser, source_table)\n cmd_compress(args, source_table, client, args.table_name, keys)\n elif args.action == \"delete-pickles\":\n keys = resolve_keys(args, parser, source_table)\n cmd_delete_pickles(args, source_table, keys)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "tools/inspect_serialized_state.py", "content": "\"\"\"Read a state file or db and create a report which summarizes it's contents.\n\nDisplays:\nState configuration\nCount of jobs\n\nTable of Jobs with start date of last run\n\n\"\"\"\nimport optparse\n\nfrom tron.config import manager\nfrom tron.serialize.runstate import statemanager\nfrom tron.utils import chdir\n\n\ndef parse_options():\n parser = optparse.OptionParser()\n parser.add_option(\"-c\", \"--config-path\", help=\"Path to the configuration.\")\n parser.add_option(\n \"-w\",\n \"--working-dir\",\n default=\".\",\n help=\"Working directory to resolve relative paths.\",\n )\n opts, _ = parser.parse_args()\n\n if not opts.config_path:\n parser.error(\"A --config-path is required.\")\n return opts\n\n\ndef get_container(config_path):\n config_manager = manager.ConfigManager(config_path)\n return config_manager.load()\n\n\ndef get_state(container):\n config = container.get_master().state_persistence\n state_manager = statemanager.PersistenceManagerFactory.from_config(config)\n names = container.get_job_names()\n return state_manager.restore(*names)\n\n\ndef format_date(date_string):\n return date_string.strftime(\"%Y-%m-%d %H:%M:%S\") if date_string else None\n\n\ndef format_jobs(job_states):\n format = \"%-30s %-8s %-5s %s\\n\"\n header = format % (\"Name\", \"Enabled\", \"Runs\", \"Last Update\")\n\n def max_run(item):\n start_time = filter(None, (run[\"start_time\"] for run in item))\n return max(start_time) if start_time else None\n\n def build(name, job):\n start_times = (max_run(job_run[\"runs\"]) for job_run in job[\"runs\"])\n start_times = filter(None, start_times)\n last_run = format_date(max(start_times)) if start_times else None\n return format % (name, job[\"enabled\"], len(job[\"runs\"]), last_run)\n\n seq = sorted(build(*item) for item in job_states.items())\n return header + \"\".join(seq)\n\n\ndef display_report(state_config, job_states):\n print(\"State Config: %s\" % str(state_config))\n print(\"Total Jobs: %s\" % len(job_states))\n\n print(\"\\n%s\" % format_jobs(job_states))\n\n\ndef main(config_path, working_dir):\n container = get_container(config_path)\n config = container.get_master().state_persistence\n with chdir(working_dir):\n display_report(config, *get_state(container))\n\n\nif __name__ == \"__main__\":\n opts = parse_options()\n main(opts.config_path, opts.working_dir)\n" }, { "path": "tools/migration/migrate_config_0.2_to_0.3.py", "content": "\"\"\"\n Convert a 0.2.x Tron configuration file to the 0.3 format.\n\n Removes YAML anchors, references, and tags.\n Display warnings for NodePools under the nodes section.\n Display warnings for action requires sections that are not lists.\n\n\"\"\"\nimport optparse\nimport re\nimport sys\n\nfrom tron import yaml\n\nYAML_TAG_RE = re.compile(r\"!\\w+\\b\")\n\n\nclass Loader(yaml.Loader):\n \"\"\"A YAML loader that does not clear its anchor mapping.\"\"\"\n\n def compose_document(self):\n self.get_event()\n node = self.compose_node(None, None)\n self.get_event()\n return node\n\n\ndef strip_tags(source):\n \"\"\"Remove YAML tags.\"\"\"\n return YAML_TAG_RE.sub(\"\", source)\n\n\ndef name_from_doc(doc):\n \"\"\"Find the string identifier for a doc.\"\"\"\n if \"name\" in doc:\n return doc[\"name\"]\n\n # Special case for node without a name, their name defaults to their hostname\n if set(doc.keys()) == {\"hostname\"}:\n return doc[\"hostname\"]\n\n if set(doc.keys()) == {\"nodes\"}:\n raise ValueError(\"Please create a name for NodePool %s\" % doc)\n\n raise ValueError(\"Could not find a name for %s\" % doc)\n\n\ndef warn_node_pools(content):\n doc = yaml.safe_load(content)\n\n node_pools = [node_doc for node_doc in doc[\"nodes\"] if \"nodes\" in node_doc]\n\n if not node_pools:\n return\n\n print(\n \"\\n\\nNode Pools should be moved into a node_pools section.\"\n + \" The following node pools were found:\\n\"\n + \"\\n\".join(str(n) for n in node_pools),\n file=sys.stderr,\n )\n\n\ndef warn_requires_list(content):\n action_names = []\n doc = yaml.safe_load(content)\n\n for job in doc[\"jobs\"]:\n for action in job[\"actions\"]:\n if \"requires\" not in action:\n continue\n\n if isinstance(action[\"requires\"], list):\n continue\n\n action_names.append(\"{}.{}\".format(job[\"name\"], action[\"name\"]))\n\n if not action_names:\n return\n\n print(\n \"\\n\\nAction requires should be a list.\"\n + \" The following actions have requires that are not lists:\\n\"\n + \"\\n\".join(action_names),\n file=sys.stderr,\n )\n\n\ndef create_loader(content):\n \"\"\"Create a loader, and have it create the document from content.\"\"\"\n loader = Loader(content)\n loader.get_single_node()\n return loader\n\n\ndef build_anchor_mapping(content):\n \"\"\"Return a map of anchors to the new name to use.\"\"\"\n loader = create_loader(content)\n\n return {\n anchor_name: name_from_doc(loader.construct_document(yaml_node))\n for anchor_name, yaml_node in loader.anchors.items()\n }\n\n\ndef update_references(content):\n anchor_mapping = build_anchor_mapping(content)\n\n def key_length_func(kv):\n return len(kv[0])\n\n anchors_by_length = sorted(\n anchor_mapping.items(),\n key=key_length_func,\n reverse=True,\n )\n for anchor_name, string_name in anchors_by_length:\n # Remove the anchors\n content = re.sub(r\"\\s*&%s ?\" % anchor_name, \"\", content)\n # Update the reference to use the string identifier\n content = re.sub(\n r\"\\*%s\\b\" % anchor_name,\n '\"%s\"' % string_name,\n content,\n )\n\n return content\n\n\ndef convert(source, dest):\n with open(source) as fh:\n content = fh.read()\n\n try:\n content = strip_tags(content)\n content = update_references(content)\n warn_node_pools(content)\n warn_requires_list(content)\n except yaml.scanner.ScannerError as e:\n print(f\"Bad content: {e}\\n{content}\")\n\n with open(dest, \"w\") as fh:\n fh.write(content)\n\n\nif __name__ == \"__main__\":\n opt_parser = optparse.OptionParser()\n opt_parser.add_option(\"-s\", dest=\"source\", help=\"Source config filename.\")\n opt_parser.add_option(\"-d\", dest=\"dest\", help=\"Destination filename.\")\n opts, args = opt_parser.parse_args()\n if not opts.source or not opts.dest:\n print(\"Source and destination filenames required.\", file=sys.stderr)\n sys.exit(1)\n\n convert(opts.source, opts.dest)\n" }, { "path": "tools/migration/migrate_config_0.5.1_to_0.5.2.py", "content": "\"\"\"Migrate a single configuration file (tron 0.5.1) to the new 0.5.2\nmulti-file format.\n\nUsage:\n\npython tools/migration/migrate_config_0.5.1_to_0.5.2.py \\\n --source old_config_filename \\\n --dest new_config_dir\n\"\"\"\nimport optparse\nimport os\n\nfrom tron.config import manager\n\n\ndef parse_options():\n parser = optparse.OptionParser()\n parser.add_option(\"-s\", \"--source\", help=\"Path to old configuration file.\")\n parser.add_option(\n \"-d\",\n \"--dest\",\n help=\"Path to new configuration directory.\",\n )\n opts, _ = parser.parse_args()\n\n if not opts.source:\n parser.error(\"--source is required\")\n if not opts.dest:\n parser.error(\"--dest is required\")\n return opts\n\n\ndef main(source, dest):\n dest = os.path.abspath(dest)\n if not os.path.isfile(source):\n raise SystemExit(\"Error: Source (%s) is not a file\" % source)\n if os.path.exists(dest):\n raise SystemExit(\"Error: Destination path (%s) already exists\" % dest)\n old_config = manager.read_raw(source)\n manager.create_new_config(dest, old_config)\n\n\nif __name__ == \"__main__\":\n opts = parse_options()\n main(opts.source, opts.dest)\n" }, { "path": "tools/migration/migrate_state.py", "content": "\"\"\"\n Migrate a state file/database from one StateStore implementation to another. It\n may also be used to add namespace names to jobs when upgrading\n from pre-0.5.2 to version 0.5.2.\n\n Usage:\n python tools/migration/migrate_state.py \\\n -s -d [ --namespace ]\n\n old_config.yaml and new_config.yaml should be configuration files with valid\n state_persistence sections. The state_persistence section configures the\n StateStore.\n\n Pre 0.5 state files can be read by the YamlStateStore. See the configuration\n documentation for more details on how to create state_persistence sections.\n\"\"\"\nimport optparse\n\nfrom tron.config import manager\nfrom tron.config import schema\nfrom tron.serialize import runstate\nfrom tron.serialize.runstate.statemanager import PersistenceManagerFactory\nfrom tron.utils import chdir\n\n\ndef parse_options():\n parser = optparse.OptionParser()\n parser.add_option(\n \"-s\",\n \"--source\",\n help=\"The source configuration path which contains a state_persistence \"\n \"section configured for the state file/database.\",\n )\n parser.add_option(\n \"-d\",\n \"--dest\",\n help=\"The destination configuration path which contains a \"\n \"state_persistence section configured for the state file/database.\",\n )\n parser.add_option(\n \"--source-working-dir\",\n help=\"The working directory for source dir to resolve relative paths.\",\n )\n parser.add_option(\n \"--dest-working-dir\",\n help=\"The working directory for dest dir to resolve relative paths.\",\n )\n parser.add_option(\n \"--namespace\",\n action=\"store_true\",\n help=\"Move jobs which are missing a namespace to the MASTER\",\n )\n\n opts, args = parser.parse_args()\n\n if not opts.source:\n parser.error(\"--source is required\")\n if not opts.dest:\n parser.error(\"--dest is required.\")\n\n return opts, args\n\n\ndef get_state_manager_from_config(config_path, working_dir):\n \"\"\"Return a state manager from the configuration.\"\"\"\n config_manager = manager.ConfigManager(config_path)\n config_container = config_manager.load()\n state_config = config_container.get_master().state_persistence\n with chdir(working_dir):\n return PersistenceManagerFactory.from_config(state_config)\n\n\ndef get_current_config(config_path):\n config_manager = manager.ConfigManager(config_path)\n return config_manager.load()\n\n\ndef add_namespaces(state_data):\n return {f\"{schema.MASTER_NAMESPACE}.{name}\": data for (name, data) in state_data.items()}\n\n\ndef strip_namespace(names):\n return [name.split(\".\", 1)[1] for name in names]\n\n\ndef convert_state(opts):\n source_manager = get_state_manager_from_config(\n opts.source,\n opts.source_working_dir,\n )\n dest_manager = get_state_manager_from_config(\n opts.dest,\n opts.dest_working_dir,\n )\n container = get_current_config(opts.source)\n\n msg = \"Migrating state from %s to %s\"\n print(msg % (source_manager._impl, dest_manager._impl))\n\n job_names = container.get_job_names()\n if opts.namespace:\n job_names = strip_namespace(job_names)\n\n job_states = source_manager.restore(\n job_names,\n )\n source_manager.cleanup()\n\n if opts.namespace:\n job_states = add_namespaces(job_states)\n\n for name, job in job_states.items():\n dest_manager.save(runstate.JOB_STATE, name, job)\n print(\"Migrated %s jobs.\" % len(job_states))\n\n dest_manager.cleanup()\n\n\nif __name__ == \"__main__\":\n opts, _args = parse_options()\n convert_state(opts)\n" }, { "path": "tools/migration/migrate_state_1.3.15_to_1.4.0.py", "content": "import argparse\nimport logging\n\nfrom tron.config import manager\nfrom tron.serialize import runstate\nfrom tron.serialize.runstate.statemanager import PersistenceManagerFactory\nfrom tron.utils import chdir\n\n\ndef parse_args():\n parser = argparse.ArgumentParser()\n parser.add_argument(\n \"--back\",\n help=\"Flag to migrate back from new state back to old state\",\n action=\"store_true\",\n default=False,\n )\n parser.add_argument(\n \"--working-dir\",\n help=\"Working directory for the Tron daemon\",\n required=True,\n )\n parser.add_argument(\n \"--config-path\",\n help=\"Path in working dir with configs\",\n required=True,\n )\n return parser.parse_args()\n\n\ndef create_job_runs_for_job(state_manager, job_name, job_state):\n for run in job_state[\"runs\"]:\n run_num = run[\"run_num\"]\n state_manager.save(runstate.JOB_RUN_STATE, f\"{job_name}.{run_num}\", run)\n run_nums = [run[\"run_num\"] for run in job_state[\"runs\"]]\n job_state[\"run_nums\"] = run_nums\n # Note: not removing 'runs' from job_state for safety.\n # If Tron starts up correctly after the state migration, it will update the job state\n # and remove 'runs'.\n state_manager.save(runstate.JOB_STATE, job_name, job_state)\n\n\ndef move_job_runs_to_job(state_manager, job_name, job_state):\n runs = state_manager._restore_runs_for_job(job_name, job_state)\n job_state[\"runs\"] = runs\n state_manager.save(runstate.JOB_STATE, job_name, job_state)\n for run in runs:\n state_manager.delete(runstate.JOB_RUN_STATE, f'{job_name}.{run[\"run_num\"]}')\n\n\ndef update_state(state_manager, job_names, back):\n jobs = state_manager._restore_dicts(runstate.JOB_STATE, job_names)\n for job_name, job_state in jobs.items():\n if back:\n move_job_runs_to_job(state_manager, job_name, job_state)\n else:\n create_job_runs_for_job(state_manager, job_name, job_state)\n\n\ndef migrate_state(config_path, working_dir, back):\n with chdir(working_dir):\n config_manager = manager.ConfigManager(config_path)\n config_container = config_manager.load()\n job_names = config_container.get_job_names()\n state_config = config_container.get_master().state_persistence\n state_manager = PersistenceManagerFactory.from_config(state_config)\n update_state(state_manager, job_names, back)\n state_manager.cleanup()\n\n\nif __name__ == \"__main__\":\n # INFO for boto, DEBUG for all tron-related state logs\n logging.basicConfig(level=logging.INFO)\n logging.getLogger(\"tron\").setLevel(logging.DEBUG)\n\n args = parse_args()\n migrate_state(args.config_path, args.working_dir, args.back)\n" }, { "path": "tools/pickles_to_json.py", "content": "import argparse\nimport math\nimport os\nimport pickle\n\nimport boto3\nimport requests\nfrom boto3.resources.base import ServiceResource\n\nfrom tron.core.job import Job\nfrom tron.core.jobrun import JobRun\nfrom tron.serialize import runstate\n\n# Max DynamoDB object size is 400KB. Since we save two copies of the object (pickled and JSON),\n# we need to consider this max size applies to the entire item, so we use a max size of 200KB\n# for each version.\nOBJECT_SIZE = 200_000\n\n\ndef get_dynamodb_table(\n aws_profile: str | None = None, table: str = \"infrastage-tron-state\", region: str = \"us-west-1\"\n) -> ServiceResource:\n \"\"\"\n Get the DynamoDB table resource.\n :param aws_profile: The name of the AWS profile to use.\n :param table: The name of the table to get.\n :param region: The region of the table.\n :return: The DynamoDB table resource.\n \"\"\"\n session = boto3.Session(profile_name=aws_profile) if aws_profile else boto3.Session()\n return session.resource(\"dynamodb\", region_name=region).Table(table)\n\n\ndef get_all_jobs(source_table: ServiceResource) -> list[str]:\n \"\"\"\n Scan the DynamoDB table and return a list of unique job keys.\n :param source_table: The DynamoDB table resource to scan.\n :return: A list of all job keys.\n \"\"\"\n items = scan_table(source_table)\n unique_keys = {item.get(\"key\", \"Unknown Key\") for item in items}\n return list(unique_keys)\n\n\ndef get_job_names(base_url: str) -> list[str]:\n \"\"\"\n Get the list of job names from the Tron API.\n :param base_url: The base URL of the Tron API.\n :return: A list of job names.\n \"\"\"\n try:\n full_url = f\"http://{base_url}.yelpcorp.com:8089/api/jobs?include_job_runs=0\"\n response = requests.get(full_url)\n response.raise_for_status()\n data = response.json()\n job_names = [job[\"name\"] for job in data.get(\"jobs\", [])]\n return job_names\n except requests.exceptions.RequestException as e:\n print(f\"An error occurred: {e}\")\n return []\n\n\ndef combine_pickle_partitions(source_table: ServiceResource, key: str) -> bytes:\n \"\"\"\n Load and combine all partitions of a pickled item from DynamoDB.\n :param source_table: The DynamoDB table resource.\n :param key: The primary key of the item to retrieve.\n :return: The combined pickled data as bytes.\n \"\"\"\n response = source_table.get_item(Key={\"key\": key, \"index\": 0}, ConsistentRead=True)\n if \"Item\" not in response:\n raise Exception(f\"No item found for key {key} at index 0\")\n item = response[\"Item\"]\n num_partitions = int(item.get(\"num_partitions\", 1))\n combined_data = bytearray()\n for index in range(num_partitions):\n response = source_table.get_item(Key={\"key\": key, \"index\": index}, ConsistentRead=True)\n if \"Item\" not in response:\n raise Exception(f\"Missing partition {index} for key {key}\")\n item = response[\"Item\"]\n combined_data.extend(item[\"val\"].value)\n return bytes(combined_data)\n\n\ndef dump_pickle_key(source_table: ServiceResource, key: str) -> None:\n \"\"\"\n Load the pickled data from DynamoDB for a given key, handling partitioned\n items, and print the full pickle data.\n :param source_table: The DynamoDB table resource.\n :param key: The primary key of the item to retrieve.\n \"\"\"\n try:\n pickled_data = combine_pickle_partitions(source_table, key)\n loaded_pickle = pickle.loads(pickled_data)\n print(f\"Key: {key} - Pickle data:\")\n print(loaded_pickle)\n except Exception as e:\n print(f\"Key: {key} - Failed to load pickle: {e}\")\n raise\n\n\ndef dump_pickle_keys(source_table: ServiceResource, keys: list[str]) -> None:\n \"\"\"\n Load and print pickles for the given list of keys.\n :param source_table: The DynamoDB table resource.\n :param keys: A list of keys for which to load and print pickles.\n \"\"\"\n for key in keys:\n dump_pickle_key(source_table, key)\n\n\ndef dump_json_key(source_table: ServiceResource, key: str) -> None:\n \"\"\"\n Load the JSON data from DynamoDB for a given key and print it.\n :param source_table: The DynamoDB table resource.\n :param key: The primary key of the item to retrieve.\n \"\"\"\n try:\n json_data = combine_json_partitions(source_table, key)\n if json_data is not None:\n print(f\"Key: {key} - JSON data:\")\n print(json_data)\n else:\n print(f\"Key: {key} - No JSON value found\")\n except Exception as e:\n print(f\"Key: {key} - Failed to load JSON: {e}\")\n\n\ndef dump_json_keys(source_table: ServiceResource, keys: list[str]) -> None:\n \"\"\"\n Load and print JSON data for the given list of keys.\n :param source_table: The DynamoDB table resource.\n :param keys: A list of keys for which to load and print JSON data.\n \"\"\"\n for key in keys:\n dump_json_key(source_table, key)\n\n\n# TODO: clean up old run history for valid jobs? something something look at job_state, then whitelist those runs instead of whitelisting entire jobs\ndef delete_keys(source_table: ServiceResource, keys: list[str]) -> None:\n \"\"\"\n Delete items with the given list of keys from the DynamoDB table.\n :param source_table: The DynamoDB table resource.\n :param keys: A list of keys to delete.\n \"\"\"\n total_keys = len(keys)\n deleted_count = 0\n failure_count = 0\n for key in keys:\n try:\n num_partitions = get_num_partitions(source_table, key)\n for index in range(num_partitions):\n source_table.delete_item(Key={\"key\": key, \"index\": index})\n print(f\"Key: {key} - Successfully deleted\")\n deleted_count += 1\n except Exception as e:\n print(f\"Key: {key} - Failed to delete: {e}\")\n failure_count += 1\n print(f\"Total keys: {total_keys}\")\n print(f\"Successfully deleted: {deleted_count}\")\n print(f\"Failures: {failure_count}\")\n\n\ndef get_num_partitions(source_table: ServiceResource, key: str) -> int:\n \"\"\"\n Get the number of partitions for a given key in the DynamoDB table.\n :param source_table: The DynamoDB table resource.\n :param key: The primary key of the item to retrieve.\n :return: The number of partitions for the key.\n \"\"\"\n response = source_table.get_item(Key={\"key\": key, \"index\": 0}, ConsistentRead=True)\n if \"Item\" not in response:\n return 0\n item = response[\"Item\"]\n num_partitions = int(item.get(\"num_partitions\", 1))\n num_json_val_partitions = int(item.get(\"num_json_val_partitions\", 0))\n return max(num_partitions, num_json_val_partitions)\n\n\ndef combine_json_partitions(source_table: ServiceResource, key: str) -> str | None:\n \"\"\"\n Combine all partitions of a JSON item from DynamoDB.\n :param source_table: The DynamoDB table resource.\n :param key: The primary key of the item to retrieve.\n :return: The combined JSON data as a string, or None if not found.\n \"\"\"\n response = source_table.get_item(Key={\"key\": key, \"index\": 0}, ConsistentRead=True)\n if \"Item\" not in response:\n return None\n item = response[\"Item\"]\n num_json_partitions = int(item.get(\"num_json_val_partitions\", 0))\n if num_json_partitions == 0:\n return None\n combined_json = \"\"\n for index in range(num_json_partitions):\n response = source_table.get_item(Key={\"key\": key, \"index\": index}, ConsistentRead=True)\n if \"Item\" not in response:\n raise Exception(f\"Missing JSON partition {index} for key {key}\")\n item = response[\"Item\"]\n if \"json_val\" in item:\n combined_json += item[\"json_val\"]\n else:\n raise Exception(f\"No 'json_val' in partition {index} for key {key}\")\n return combined_json\n\n\ndef convert_pickle_to_json_and_update_table(source_table: ServiceResource, key: str, dry_run: bool = True) -> bool:\n \"\"\"\n Convert a single pickled item to JSON and update the DynamoDB entry.\n Returns True if the conversion was successful, False if skipped.\n Raises an exception if conversion fails.\n \"\"\"\n try:\n # Skip conversion for job_state MASTER and job_run_state MASTER jobs that are from infrastage testing (i.e., not real jobs)\n if key.startswith(\"job_state MASTER\") or key.startswith(\"job_run_state MASTER\"):\n print(f\"Skipping conversion for key: {key}\")\n return False\n pickled_data = combine_pickle_partitions(source_table, key)\n state_data = pickle.loads(pickled_data)\n state_type = key.split()[0]\n if state_type == runstate.JOB_STATE:\n json_data = Job.to_json(state_data)\n elif state_type == runstate.JOB_RUN_STATE:\n json_data = JobRun.to_json(state_data)\n else:\n # This will skip the state metadata and any other non-standard keys we have in the table\n print(f\"Key: {key} - Unknown state type: {state_type}. Skipping.\")\n return False\n num_json_partitions = math.ceil(len(json_data) / OBJECT_SIZE)\n for partition_index in range(num_json_partitions):\n json_partition = json_data[\n partition_index * OBJECT_SIZE : min((partition_index + 1) * OBJECT_SIZE, len(json_data))\n ]\n if not dry_run:\n source_table.update_item(\n Key={\"key\": key, \"index\": partition_index},\n UpdateExpression=\"SET json_val = :json, num_json_val_partitions = :num_partitions\",\n ExpressionAttributeValues={\n \":json\": json_partition,\n \":num_partitions\": num_json_partitions,\n },\n )\n if dry_run:\n print(f\"DRY RUN: Key: {key} - Pickle would have been converted to JSON and updated\")\n else:\n print(f\"Key: {key} - Pickle converted to JSON and updated\")\n return True\n except Exception as e:\n print(f\"Key: {key} - Failed to convert pickle to JSON: {e}\")\n raise\n\n\ndef convert_pickles_to_json_and_update_table(\n source_table: ServiceResource,\n keys: list[str],\n dry_run: bool = True,\n deprecated_keys_output: str | None = None,\n failed_keys_output: str | None = None,\n job_names: list[str] = [],\n) -> None:\n \"\"\"\n Convert pickled items in the DynamoDB table to JSON and update the entries.\n :param source_table: The DynamoDB table resource.\n :param keys: List of keys to convert.\n :param dry_run: If True, simulate the conversion without updating the table.\n :param deprecated_keys_output: Output file to write deprecated keys to.\n :param failed_keys_output: Output file to write keys that failed to convert to.\n :param job_names: List of job names to use for filtering keys.\n \"\"\"\n total_keys = len(keys)\n converted_keys = 0\n skipped_keys = 0\n failed_keys = []\n delete_keys = []\n\n for key in keys:\n # Extract the job name from the key\n parts = key.split()\n if len(parts) < 2:\n continue\n\n state_type, job_info = parts[0], parts[1]\n\n # Ignore run_num for job_run_state keys\n if state_type == \"job_run_state\":\n job_name = \".\".join(job_info.split(\".\")[:-1])\n else:\n job_name = job_info\n\n if job_name not in job_names:\n delete_keys.append(key)\n continue\n\n try:\n result = convert_pickle_to_json_and_update_table(source_table, key, dry_run)\n if result:\n converted_keys += 1\n else:\n skipped_keys += 1\n except Exception as e:\n print(f\"Key: {key} - Failed to convert pickle to JSON: {e}\")\n failed_keys.append(key)\n\n print(f\"Total keys processed: {total_keys}\")\n print(f\"Conversions attempted: {total_keys - skipped_keys}\")\n print(f\"Conversions succeeded: {converted_keys}\")\n print(f\"Conversions skipped: {skipped_keys}\")\n print(f\"Conversions failed: {len(failed_keys)}\")\n print(f\"Keys to be deleted: {len(delete_keys)}\")\n\n if deprecated_keys_output:\n with open(deprecated_keys_output, \"w\") as f:\n for key in delete_keys:\n f.write(f\"{key}\\n\")\n print(f\"Deprecated keys have been written to {deprecated_keys_output}\")\n if failed_keys_output:\n with open(failed_keys_output, \"w\") as f:\n for key in failed_keys:\n f.write(f\"{key}\\n\")\n print(f\"Failed have been written to {failed_keys_output}\")\n if dry_run:\n print(\"Dry run complete. No changes were made to the DynamoDB table.\")\n\n\ndef scan_table(source_table: ServiceResource) -> list[dict]:\n \"\"\"\n Scan the DynamoDB table and return all items, handling pagination.\n :param source_table: The DynamoDB table resource to scan.\n :return: A list of all items in the table.\n \"\"\"\n items = []\n response = source_table.scan()\n items.extend(response.get(\"Items\", []))\n while \"LastEvaluatedKey\" in response:\n response = source_table.scan(ExclusiveStartKey=response[\"LastEvaluatedKey\"])\n items.extend(response.get(\"Items\", []))\n return items\n\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"Utilities for working with pickles and JSON items in Tron's DynamoDB state store.\",\n epilog=\"\"\"\nActions:\n convert Convert pickled state data to JSON format and update the DynamoDB table.\n dump-pickle Load and print the pickles for specified keys.\n dump-json Load and print JSON data for specified keys.\n delete Delete the specified keys from the DynamoDB table.\nExamples:\n Validate pickles (write deprecated keys to deprecated_keys.txt, dry run by default):\n pickles_to_json.py --table-name infrastage-tron-state --table-region us-west-1 convert --all --deprecated-keys-output deprecated_keys.txt --tron-api tron-infrastage\n Convert all pickles to JSON (actually execute the update):\n pickles_to_json.py --table-name infrastage-tron-state --table-region us-west-1 convert --all --execute --tron-api tron-infrastage\n Convert specific pickles to JSON using keys from an input file:\n pickles_to_json.py --table-name infrastage-tron-state --table-region us-west-1 convert --keys-file input_keys.txt --execute --tron-api tron-infrastage\n Load and print specific JSON keys using keys from an input file:\n pickles_to_json.py --table-name infrastage-tron-state --table-region us-west-1 dump-json --keys-file input_keys.txt\n Delete specific keys (dry run by default):\n pickles_to_json.py --table-name infrastage-tron-state --table-region us-west-1 delete --keys \"key1\" \"key2\"\n Delete keys from an input file (actually execute the deletion):\n pickles_to_json.py --table-name infrastage-tron-state --table-region us-west-1 delete --keys-file input_keys.txt --execute\n\"\"\",\n formatter_class=argparse.RawDescriptionHelpFormatter,\n )\n parser.add_argument(\n \"--aws-profile\",\n default=os.environ.get(\"AWS_PROFILE\", None),\n help=\"AWS profile to use (default: taken from AWS_PROFILE environment variable)\",\n )\n parser.add_argument(\"--table-name\", required=True, help=\"Name of the DynamoDB table\")\n parser.add_argument(\"--table-region\", required=True, help=\"AWS region of the DynamoDB table\")\n\n subparsers = parser.add_subparsers(dest=\"action\", required=True, help=\"Action to perform\")\n\n convert_parser = subparsers.add_parser(\n \"convert\", help=\"Convert pickled state data to JSON format and update the DynamoDB table.\"\n )\n convert_parser.add_argument(\n \"--tron-api\",\n required=True,\n help=\"Base URL of the Tron API to fetch job names from.\",\n )\n convert_parser.add_argument(\n \"--keys\",\n nargs=\"+\",\n required=False,\n help=\"Specific key(s) to perform the action on.\",\n )\n convert_parser.add_argument(\n \"--keys-file\",\n required=False,\n help=\"Input file containing keys to perform the action on. One key per line.\",\n )\n convert_parser.add_argument(\n \"--deprecated-keys-output\",\n required=False,\n help=\"Output file to write deprecated keys to. These are keys associated with jobs not present in Tron. One key per line.\",\n )\n convert_parser.add_argument(\n \"--failed-keys-output\",\n required=False,\n help=\"Output file to write keys that failed to convert. One key per line.\",\n )\n convert_parser.add_argument(\n \"--all\",\n action=\"store_true\",\n help=\"Apply the action to all keys in the table.\",\n )\n convert_parser.add_argument(\n \"--execute\",\n action=\"store_true\",\n default=False,\n help=\"Actually perform the conversion and update the DynamoDB table.\",\n )\n\n dump_pickle_parser = subparsers.add_parser(\"dump-pickle\", help=\"Load and print the pickles for specified keys.\")\n dump_pickle_parser.add_argument(\n \"--keys\",\n nargs=\"+\",\n required=False,\n help=\"Specific key(s) to perform the action on.\",\n )\n dump_pickle_parser.add_argument(\n \"--keys-file\",\n required=False,\n help=\"Input file containing keys to perform the action on. One key per line.\",\n )\n dump_pickle_parser.add_argument(\n \"--all\",\n action=\"store_true\",\n help=\"Apply the action to all keys in the table.\",\n )\n\n dump_json_parser = subparsers.add_parser(\"dump-json\", help=\"Load and print JSON data for specified keys.\")\n dump_json_parser.add_argument(\n \"--keys\",\n nargs=\"+\",\n required=False,\n help=\"Specific key(s) to perform the action on.\",\n )\n dump_json_parser.add_argument(\n \"--keys-file\",\n required=False,\n help=\"Input file containing keys to perform the action on. One key per line.\",\n )\n dump_json_parser.add_argument(\n \"--all\",\n action=\"store_true\",\n help=\"Apply the action to all keys in the table.\",\n )\n\n delete_keys_parser = subparsers.add_parser(\"delete\", help=\"Delete the specified keys from the DynamoDB table.\")\n delete_keys_parser.add_argument(\n \"--keys\",\n nargs=\"+\",\n required=False,\n help=\"Specific key(s) to perform the action on.\",\n )\n delete_keys_parser.add_argument(\n \"--keys-file\",\n required=False,\n help=\"Input file containing keys to perform the action on. One key per line.\",\n )\n delete_keys_parser.add_argument(\n \"--all\",\n action=\"store_true\",\n help=\"Apply the action to all keys in the table.\",\n )\n delete_keys_parser.add_argument(\n \"--execute\",\n action=\"store_true\",\n default=False,\n help=\"Actually perform the deletion on the DynamoDB table.\",\n )\n\n args = parser.parse_args()\n source_table = get_dynamodb_table(args.aws_profile, args.table_name, args.table_region)\n\n if not args.keys and not args.keys_file and not args.all:\n parser.error(\"You must provide either --keys, --keys-file, or --all.\")\n\n if args.all:\n print(\"Processing all keys in the table...\")\n keys = get_all_jobs(source_table)\n else:\n keys = []\n if args.keys:\n keys.extend(args.keys)\n if args.keys_file:\n try:\n with open(args.keys_file) as f:\n keys_from_file = [line.strip() for line in f if line.strip()]\n keys.extend(keys_from_file)\n except Exception as e:\n parser.error(f\"Error reading keys from file {args.keys_file}: {e}\")\n if not keys:\n parser.error(\"No keys provided. Please provide keys via --keys or --keys-file.\")\n keys = list(set(keys))\n\n if args.action == \"convert\":\n job_names = get_job_names(args.tron_api)\n convert_pickles_to_json_and_update_table(\n source_table,\n keys=keys,\n dry_run=not args.execute,\n deprecated_keys_output=args.deprecated_keys_output,\n failed_keys_output=args.failed_keys_output,\n job_names=job_names,\n )\n elif args.action == \"dump-pickle\":\n dump_pickle_keys(source_table, keys)\n elif args.action == \"dump-json\":\n dump_json_keys(source_table, keys)\n elif args.action == \"delete\":\n if not args.execute:\n print(f\"DRY RUN: Would delete {len(keys)} keys from the table '{args.table_name}'.\")\n for key in keys:\n print(f\"Would delete key: {key}\")\n print(\"Dry run complete. No changes were made to the DynamoDB table.\")\n else:\n confirm = (\n input(f\"Are you sure you want to delete {len(keys)} keys from the table '{args.table_name}'? [y/N]: \")\n .strip()\n .lower()\n )\n if confirm in (\"y\", \"yes\"):\n delete_keys(source_table, keys)\n else:\n print(\"Deletion cancelled.\")\n else:\n print(f\"Unknown action: {args.action}\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "tools/sync_tron_state_from_k8s.py", "content": "\"\"\"\nUpdate tron state from k8s api if tron has not yet updated correctly\n\n Usage:\n python tools/sync_tron_state_from_k8s.py -c (--do-work|--num-runs N|--tronctl-wrapper tronctl-pnw-devc)\n\nThis will search for completed pods in the cluster specified in the kubeconfig in the `tron` namespace and use tronctl to transition any whose states do not match.\n\"\"\"\nimport argparse\nimport base64\nimport hashlib\nimport logging\nimport os\nimport subprocess\nimport sys\nfrom typing import Any\n\nfrom kubernetes.client import V1Pod\nfrom task_processing.plugins.kubernetes.kube_client import KubeClient\n\nfrom tron.commands.client import Client\nfrom tron.commands.cmd_utils import get_client_config\n\nPOD_STATUS_TO_TRON_STATE = {\n \"Succeeded\": \"success\",\n \"Failed\": \"fail\",\n \"Unknown\": \"Unknown\", # This should never really happen\n}\n\nTRON_MODIFIABLE_STATES = [\n \"starting\", # stuck jobs\n \"running\", # stuck jobs\n \"unknown\",\n \"lost\",\n]\n\nlog = logging.getLogger(\"sync_tron_from_k8s\")\n\n\n# NOTE: Copied from paasta_tools.kubernetes_tools, if it changes there it must be updated here\ndef limit_size_with_hash(name: str, limit: int = 63, suffix: int = 4) -> str:\n \"\"\"Returns `name` unchanged if it's length does not exceed the `limit`.\n Otherwise, returns truncated `name` with it's hash of size `suffix`\n appended.\n\n base32 encoding is chosen as it satisfies the common requirement in\n various k8s names to be alphanumeric.\n \"\"\"\n if len(name) > limit:\n digest = hashlib.md5(name.encode()).digest()\n hash = base64.b32encode(digest).decode().replace(\"=\", \"\").lower()\n return f\"{name[:(limit-suffix-1)]}-{hash[:suffix]}\"\n else:\n return name\n\n\ndef parse_args():\n parser = argparse.ArgumentParser()\n parser.add_argument(\n \"--kubeconfig-path\",\n dest=\"kubeconfig_path\",\n help=\"KUBECONFIG path; multiple can be specified to find pods in multiple clusters\",\n nargs=\"+\",\n )\n parser.add_argument(\n \"--kubecontext\",\n dest=\"kubecontext\",\n help=\"kubecontext to use from specified kubeconfig. multiple can be specified to find pods in multiple clusters, ONLY if a single kubeconfig-path is provided\",\n nargs=\"*\",\n )\n parser.add_argument(\n \"--do-work\",\n dest=\"do_work\",\n action=\"store_true\",\n default=False,\n help=\"Actually modify tron actions that need updating; without this flag we will only print those that would be updated\",\n )\n parser.add_argument(\"--tron-url\", default=None, help=\"Tron url (default will read from paasta tron config)\")\n parser.add_argument(\n \"--tronctl-wrapper\",\n default=\"tronctl\",\n dest=\"tronctl_wrapper\",\n help=\"Tronctl wrapper to use (will not use wrapper by default)\",\n )\n parser.add_argument(\"-n\", \"--num-runs\", dest=\"num_runs\", default=100, help=\"Maximum number of job runs to retrieve\")\n parser.add_argument(\"-v\", \"--verbose\", dest=\"verbose\", action=\"store_true\", default=False, help=\"Verbose logging\")\n args = parser.parse_args()\n\n # We can only have multiple kubeconfigs, or multiple contexts with a single config\n if len(args.kubeconfig_path) > 1 and args.kubecontext:\n parser.error(\"You can only specify a single --kubeconfig-path if specifying --kubecontext arguments.\")\n\n # tron's base level is critical, not info, adjust accoringly\n if args.verbose:\n level = logging.DEBUG\n tron_level = logging.WARN\n else:\n level = logging.INFO\n tron_level = logging.CRITICAL\n\n logging.basicConfig(level=level, stream=sys.stdout)\n\n tron_client_logger = logging.getLogger(\"tron.commands.client\")\n tron_client_logger.setLevel(tron_level)\n\n # We also don't want kube_client debug logs\n kube_logger = logging.getLogger(\"kubernetes.client.rest\")\n kube_logger.setLevel(logging.INFO)\n\n return args\n\n\ndef fetch_pods(kubeconfig_path: str, kubecontext: str | None) -> dict[str, V1Pod]:\n if kubecontext:\n # KubeClient only uses the environment variable\n os.environ[\"KUBECONTEXT\"] = kubecontext\n kube_client = KubeClient(kubeconfig_path=kubeconfig_path, user_agent=\"sync_tron_state_from_k8s\")\n\n # Bit of a hack, no helper to fetch pods so reach into core api\n completed_pod_list = kube_client.core.list_namespaced_pod(\n namespace=\"tron\",\n )\n\n return {pod.metadata.name: pod for pod in completed_pod_list.items}\n\n\ndef get_tron_state_from_api(tron_server: str, num_runs: int = 100) -> list[dict[str, dict[Any, Any]]]:\n if not tron_server:\n client_config = get_client_config()\n tron_server = client_config.get(\"server\", \"http://localhost:8089\")\n client = Client(tron_server)\n # /jobs returns only the latest 5 runs, we'll need to request all runs instead ourselves\n jobs = client.jobs(\n include_job_runs=False,\n include_action_runs=False,\n include_action_graph=False,\n include_node_pool=False,\n )\n\n for job in jobs:\n # Update job URL to be used with API instead of web\n url = f'/api{job[\"url\"]}'\n log.debug(f'Fetching job {job[\"name\"]} at {url}')\n job_runs = client.job(\n url,\n include_action_runs=True,\n count=num_runs, # TODO: fetch job run_limit and use that for count ?\n )\n job[\"runs\"] = job_runs[\"runs\"]\n return jobs\n\n\ndef get_matching_pod(action_run: dict[str, Any], pods: dict[str, V1Pod]) -> V1Pod | None:\n \"\"\"Given a tron action_run, try to find the right pod that matches.\"\"\"\n action_name = action_run[\"action_name\"]\n job_name = action_run[\"job_name\"]\n run_num = action_run[\"run_num\"]\n service, job = job_name.split(\".\")\n instance_name = f\"{job}.{action_name}\"\n sanitized_instance_name = limit_size_with_hash(instance_name)\n matching_pods = sorted(\n [\n pod\n for pod in pods.values()\n if pod.metadata.labels[\"paasta.yelp.com/service\"] == service\n and pod.metadata.labels[\"paasta.yelp.com/instance\"] == sanitized_instance_name\n and pod.metadata.labels[\"tron.yelp.com/run_num\"] == run_num\n ],\n # If action has retries, there will be multiple pods w/ same job_run; we only want the latest\n key=lambda pod: pod.metadata.creation_timestamp,\n reverse=True,\n )\n return (\n matching_pods[0] if matching_pods and matching_pods[0].status.phase in POD_STATUS_TO_TRON_STATE.keys() else None\n )\n\n\ndef get_desired_state_from_pod(pod: V1Pod) -> str:\n k8s_state = pod.status.phase\n return POD_STATUS_TO_TRON_STATE.get(k8s_state, \"NoMatch\")\n\n\ndef update_tron_from_pods(\n jobs: list[dict[str, Any]], pods: dict[str, V1Pod], tronctl_wrapper: str = \"tronctl\", do_work: bool = False\n):\n updated = []\n error = []\n for job in jobs:\n if job[\"runs\"]:\n # job_runs\n for job_run in job[\"runs\"]:\n # actions for this job_run\n for action in job_run.get(\"runs\", []):\n action_run_id = action[\"id\"]\n if action[\"state\"] in TRON_MODIFIABLE_STATES:\n pod = get_matching_pod(action, pods)\n if pod:\n desired_state = get_desired_state_from_pod(pod)\n if action[\"state\"] != desired_state:\n log.debug(f'{action_run_id} state {action[\"state\"]} needs updating to {desired_state}')\n cmd = [tronctl_wrapper, desired_state, action_run_id]\n if do_work:\n # tronctl-$cluster success/fail svc.job.run.action\n try:\n log.info(f\"Running {cmd}\")\n proc = subprocess.run(cmd, capture_output=True, text=True)\n if proc.returncode != 0:\n log.error(f\"Got non-zero exit code: {proc.returncode}\")\n log.error(f\"\\t{proc.stderr}\")\n error.append(action_run_id)\n updated.append(action_run_id)\n except Exception:\n log.exception(\"ERROR: Hit exception:\")\n error.append(action_run_id)\n else:\n log.info(f\"Dry-Run: Would run {cmd}\")\n updated.append(action_run_id)\n else:\n log.debug(f\"action run {action_run_id} not found in list of finished pods, no action taken\")\n else:\n log.debug(f'Action state {action[\"state\"]} for {action_run_id} not modifiable, no action taken')\n log.info(f\"Updated {len(updated)} actions: {','.join(updated)}\")\n log.info(f\"Hit {len(error)} errors on actions: {','.join(error)}\")\n return {\"updated\": updated, \"error\": error}\n\n\nif __name__ == \"__main__\":\n args = parse_args()\n\n jobs = get_tron_state_from_api(args.tron_url, args.num_runs)\n log.debug(f\"Found {len(jobs)} jobs.\")\n\n pods = {}\n kube_client_args = (\n [(args.kubeconfig_path[0], kubecontext) for kubecontext in args.kubecontext]\n if args.kubecontext\n else [(kubeconfig_path, None) for kubeconfig_path in args.kubeconfig_path]\n )\n\n for kubeconfig_path, kubecontext in kube_client_args:\n pods.update(fetch_pods(kubeconfig_path, kubecontext))\n\n log.debug(f\"Found {len(pods.keys())} pods.\")\n\n update_tron_from_pods(jobs, pods, args.tronctl_wrapper, args.do_work)\n" }, { "path": "tox.ini", "content": "[tox]\nenvlist = py310\n\n[testenv]\nbasepython = python3.10\ndeps =\n --requirement={toxinidir}/requirements.txt\n --requirement={toxinidir}/requirements-dev.txt\nusedevelop = true\npassenv = USER PIP_INDEX_URL\ncommands =\n pre-commit install -f --install-hooks\n pre-commit run --all-files\n # tron has been around for a while, so we'll need to slowly add types or make an effort\n # to get it mypy-clean in one shot - until then, let's only check files that we've added types to\n mypy --package tron\n check-requirements\n # optionally install yelpy requirements - this is after check-requirements since\n # check-requirements doesn't understand these extra requirements\n -pip install -r yelp_package/extra_requirements_yelp.txt\n # we then run tests at the very end so that we can run tests with yelpy requirements\n py.test -s {posargs:tests}\n\n[flake8]\nignore = E501,E265,E241,E704,E251,W504,E231,W503,E203\n\n[testenv:docs]\ndeps =\n --requirement={toxinidir}/requirements-docs.txt\n --requirement={toxinidir}/requirements.txt\nwhitelist_externals=\n mkdir\ncommands=\n /bin/rm -rf docs/source/generated/\n # The last arg to apidoc is a list of excluded paths\n sphinx-apidoc -f -e -o docs/source/generated/ tron\n mkdir -p docs\n sphinx-build -b html -d docs/_build docs/source docs/_build/html\n\n[testenv:itest]\ncommands =\n make deb_jammy\n make _itest_jammy\n" }, { "path": "tron/__init__.py", "content": "# Copyright 2015-2016 Yelp Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# It is imperative that this file not contain any imports from our\n# dependencies. Since this file is imported from setup.py in the\n# setup phase, the dependencies may not exist on disk yet.\n#\n# Don't bump version manually. See `make release` docs in ./Makefile\n__version__ = \"3.10.0\"\n" }, { "path": "tron/actioncommand.py", "content": "import json\nimport logging\nimport os\nfrom io import StringIO\nfrom shlex import quote\nfrom typing import Any\n\nfrom tron.config import schema\nfrom tron.serialize import filehandler\nfrom tron.utils import timeutils\nfrom tron.utils.observer import Observable\nfrom tron.utils.persistable import Persistable\nfrom tron.utils.state import Machine\n\nlog = logging.getLogger(__name__)\n\n\nclass ActionCommand(Observable):\n \"\"\"An ActionCommand encapsulates a runnable task that is passed to a node\n for execution.\n\n A Node calls:\n started (when the command starts)\n exited (when the command exits)\n write_ (when output is received)\n done (when the command is finished)\n \"\"\"\n\n PENDING = \"pending\"\n RUNNING = \"running\"\n EXITING = \"exiting\"\n COMPLETE = \"complete\"\n FAILSTART = \"failstart\"\n\n STATE_MACHINE = Machine(\n PENDING,\n **{\n PENDING: {\n \"start\": RUNNING,\n \"exit\": FAILSTART,\n },\n RUNNING: {\n \"exit\": EXITING,\n },\n EXITING: {\n \"close\": COMPLETE,\n },\n },\n )\n\n STDOUT = \".stdout\"\n STDERR = \".stderr\"\n\n def __init__(self, id, command, serializer=None):\n super().__init__()\n self.id = id\n self.command = command\n self.machine = Machine.from_machine(ActionCommand.STATE_MACHINE)\n self.exit_status = None\n self.start_time = None\n self.end_time = None\n if serializer:\n self.stdout = serializer.open(self.STDOUT)\n self.stderr = serializer.open(self.STDERR)\n else:\n self.stdout = filehandler.NullFileHandle\n self.stderr = filehandler.NullFileHandle\n\n @property\n def state(self):\n return self.machine.state\n\n def transition_and_notify(self, target):\n if self.machine.transition(target):\n self.notify(self.state)\n return True\n\n def started(self):\n if self.machine.check(\"start\"):\n self.start_time = timeutils.current_timestamp()\n return self.transition_and_notify(\"start\")\n\n def exited(self, exit_status):\n if self.machine.check(\"exit\"):\n self.end_time = timeutils.current_timestamp()\n self.exit_status = exit_status\n return self.transition_and_notify(\"exit\")\n\n def write_stderr(self, value):\n self.stderr.write(value)\n\n def write_stdout(self, value):\n self.stdout.write(value)\n\n def done(self):\n if self.machine.check(\"close\"):\n self.stdout.close()\n self.stderr.close()\n return self.transition_and_notify(\"close\")\n\n def handle_errback(self, result):\n \"\"\"Handle an unexpected error while being run. This will likely be\n an interval error. Cleanup the state of this ActionCommand and log\n something useful for debugging.\n \"\"\"\n log.error(f\"Unknown failure for {self}, {str(result)}\")\n self.exited(result)\n self.done()\n\n @property\n def is_unknown(self):\n return self.exit_status is None\n\n @property\n def is_failed(self):\n return bool(self.exit_status)\n\n @property\n def is_complete(self):\n \"\"\"Complete implies done and success.\"\"\"\n return self.machine.state == ActionCommand.COMPLETE\n\n @property\n def is_done(self):\n \"\"\"Done implies no more work will be done, but might not be success.\"\"\"\n return self.machine.state in (\n ActionCommand.COMPLETE,\n ActionCommand.FAILSTART,\n )\n\n def __repr__(self):\n return f\"ActionCommand {self.id} {self.command}: {self.state}\"\n\n\nclass StringBufferStore:\n \"\"\"A serializer object which can be passed to ActionCommand as a\n serializer, but stores streams in memory.\n \"\"\"\n\n def __init__(self):\n self.buffers = {}\n\n def open(self, name):\n return self.buffers.setdefault(name, StringIO())\n\n def clear(self):\n self.buffers.clear()\n\n\n# TODO: TRON-2304 - Cleanup NoActionRunnerFactory\nclass NoActionRunnerFactory:\n \"\"\"Action runner factory that does not wrap the action run command.\"\"\"\n\n @classmethod\n def create(cls, id, command, serializer):\n return ActionCommand(id, command, serializer)\n\n @classmethod\n def build_stop_action_command(cls, _id, _command):\n \"\"\"It is not possible to stop action commands without a runner.\"\"\"\n raise NotImplementedError(\"An action_runner is required to stop.\")\n\n @staticmethod\n def from_json():\n return None\n\n @staticmethod\n def to_json():\n return None\n\n\nclass SubprocessActionRunnerFactory(Persistable):\n \"\"\"Run actions by wrapping them in `action_runner.py`.\"\"\"\n\n runner_exec_name = \"action_runner.py\"\n status_exec_name = \"action_status.py\"\n\n def __init__(self, status_path, exec_path):\n self.status_path = status_path\n self.exec_path = exec_path\n\n @classmethod\n def from_config(cls, config):\n return cls(config.remote_status_path, config.remote_exec_path)\n\n def create(self, id, command, serializer):\n command = self.build_command(id, command, self.runner_exec_name)\n return ActionCommand(id, command, serializer)\n\n def build_command(self, id, command, exec_name):\n status_path = os.path.join(self.status_path, id)\n runner_path = os.path.join(self.exec_path, exec_name)\n return f\"{quote(runner_path)} {quote(status_path)} {quote(command)} {quote(id)}\"\n\n def build_stop_action_command(self, id, command):\n command = self.build_command(id, command, self.status_exec_name)\n run_id = f\"{id}.{command}\"\n return ActionCommand(run_id, command, StringBufferStore())\n\n def __eq__(self, other):\n return (\n self.__class__ == other.__class__\n and self.status_path == other.status_path\n and self.exec_path == other.exec_path\n )\n\n def __ne__(self, other):\n return not self == other\n\n @staticmethod\n def from_json(state_data: str) -> dict[str, Any]:\n try:\n json_data = json.loads(state_data)\n deserialized_data = {\n \"status_path\": json_data[\"status_path\"],\n \"exec_path\": json_data[\"exec_path\"],\n }\n return deserialized_data\n except Exception:\n log.exception(\"Error deserializing SubprocessActionRunnerFactory from JSON\")\n raise\n\n @staticmethod\n def to_json(state_data: dict) -> str:\n try:\n return json.dumps(\n {\n \"status_path\": state_data[\"status_path\"],\n \"exec_path\": state_data[\"exec_path\"],\n }\n )\n except KeyError:\n log.exception(\"Missing key in state_data:\")\n raise\n except Exception:\n log.exception(\"Error serializing SubprocessActionRunnerFactory to JSON:\")\n raise\n\n\ndef create_action_runner_factory_from_config(config):\n \"\"\"A factory-factory method which returns a callable that can be used to\n create ActionCommand objects. The factory definition should match the\n constructor for ActionCommand.\n \"\"\"\n if not config or config.runner_type == schema.ActionRunnerTypes.none.value:\n return NoActionRunnerFactory()\n elif config.runner_type == schema.ActionRunnerTypes.subprocess.value:\n return SubprocessActionRunnerFactory.from_config(config)\n else:\n raise ValueError(\"Unknown runner type: %s\", config.runner_type)\n" }, { "path": "tron/api/__init__.py", "content": "" }, { "path": "tron/api/adapter.py", "content": "\"\"\"\n Classes which create external representations of core objects. This allows\n the core objects to remain decoupled from the API and clients. These classes\n act as an adapter between the data format api clients expect, and the internal\n data of an object.\n\"\"\"\nimport functools\nimport os.path\nimport time\nfrom collections.abc import Callable\nfrom typing import Any\nfrom typing import TypeVar\nfrom urllib.parse import quote\n\nfrom tron import actioncommand\nfrom tron import scheduler\nfrom tron.core.actionrun import KubernetesActionRun\nfrom tron.serialize import filehandler\nfrom tron.utils import timeutils\nfrom tron.utils.logreader import read_log_stream_for_action_run\nfrom tron.utils.timeutils import delta_total_seconds\n\nR = TypeVar(\"R\")\n\n\nclass ReprAdapter:\n \"\"\"Creates a dictionary from the given object for a set of rules.\"\"\"\n\n field_names: list[str] = []\n translated_field_names: list[str] = []\n\n def __init__(self, internal_obj):\n self._obj = internal_obj\n self.fields = self._get_field_names()\n self.translators = self._get_translation_mapping()\n\n def _get_field_names(self):\n return self.field_names\n\n def _get_translation_mapping(self):\n return {field_name: getattr(self, \"get_%s\" % field_name) for field_name in self.translated_field_names}\n\n def get_repr(self):\n repr_data = {field: getattr(self._obj, field) for field in self.fields}\n translated = {field: func() for field, func in self.translators.items()}\n repr_data.update(translated)\n return repr_data\n\n\ndef adapt_many(adapter_class, seq, *args, **kwargs):\n return [adapter_class(item, *args, **kwargs).get_repr() for item in seq if item is not None]\n\n\ndef toggle_flag(\n flag_name: str,\n) -> Callable[ # the typing here is funky, this is a decorator factory that returns a decorator\n # which takes a function with \"one\" argument (really, that argument is just self) and returns some type R\n # where R is the return type of the decorated function\n [Callable[[Any], R]],\n # and that decorator returns another callable (since this is a decorator factory) that takes self (again, the Any)\n # and returns None (if the flag is False) or R (if True)\n Callable[[Any], R | None],\n]:\n \"\"\"Create a decorator which checks if flag_name is true before running\n the wrapped function. If False returns None.\n \"\"\"\n\n def wrap(f: Callable[[Any], R]) -> Callable[[Any], R | None]:\n @functools.wraps(f)\n def wrapper(self: Any) -> R | None:\n if getattr(self, flag_name):\n return f(self)\n return None\n\n return wrapper\n\n return wrap\n\n\nclass RunAdapter(ReprAdapter):\n \"\"\"Base class for JobRun and ActionRun adapters.\"\"\"\n\n def get_state(self):\n return self._obj.state\n\n def get_node(self):\n return NodeAdapter(self._obj.node).get_repr()\n\n def get_duration(self):\n duration = timeutils.duration(self._obj.start_time, self._obj.end_time)\n return str(duration or \"\")\n\n\nclass ActionRunAdapter(RunAdapter):\n \"\"\"Adapt a JobRun and an Action name to an external representation of an\n ActionRun.\n \"\"\"\n\n field_names = [\n \"id\",\n \"start_time\",\n \"end_time\",\n \"exit_status\",\n \"action_name\",\n \"exit_statuses\",\n \"retries_remaining\",\n \"original_command\",\n ]\n\n translated_field_names = [\n \"state\",\n \"node\",\n \"command\",\n \"raw_command\",\n \"requirements\",\n \"meta\",\n \"stdout\",\n \"stderr\",\n \"duration\",\n \"job_name\",\n \"run_num\",\n \"retries_delay\",\n \"in_delay\",\n \"triggered_by\",\n \"trigger_downstreams\",\n ]\n\n def __init__(\n self,\n action_run,\n job_run=None,\n max_lines=10,\n include_stdout=False,\n include_stderr=False,\n include_meta=False,\n ):\n super().__init__(action_run)\n self.job_run = job_run\n self.max_lines = max_lines or None\n self.include_stdout = include_stdout\n self.include_stderr = include_stderr\n self.include_meta = include_meta\n\n def get_raw_command(self):\n return self._obj.command_config.command\n\n def get_command(self):\n return self._obj.rendered_command\n\n @toggle_flag(\"job_run\")\n def get_requirements(self):\n action_name = self._obj.action_name\n required = self.job_run.action_graph.get_dependencies(action_name)\n return [act.name for act in required]\n\n def _get_serializer(self, path: str | None = None) -> filehandler.OutputStreamSerializer:\n base_path = filehandler.OutputPath(path) if path else self._obj.output_path\n return filehandler.OutputStreamSerializer(base_path)\n\n def _get_alternate_output_paths(self):\n try:\n namespace, jobname, run_num, action = self._obj.id.split(\".\")\n except Exception:\n return None\n\n # Check to see if the output might have ended up in any alternate locations.\n for alt_path in self._obj.STDOUT_PATHS:\n formatted_alt_path = os.path.join(\n # This ugliness is getting the \"root output directory\"\n self._obj.context.next.next.base.job.output_path.base,\n alt_path.format(\n namespace=namespace,\n jobname=jobname,\n run_num=run_num,\n action=action,\n ),\n )\n if os.path.exists(formatted_alt_path):\n yield formatted_alt_path\n\n @toggle_flag(\"include_meta\")\n def get_meta(self) -> list[str]:\n if not isinstance(self._obj, KubernetesActionRun):\n return [\"When this action is migrated to Kubernetes, this will contain Tron/task_processing output.\"]\n\n # We're reusing the \"old\" (i.e., SSH/Mesos) logging files for task_processing output since\n # that won't make it into anything but Splunk\n filename = actioncommand.ActionCommand.STDERR\n output: list[str] = self._get_serializer().tail(filename, self.max_lines)\n if not output:\n for alt_path in self._get_alternate_output_paths():\n output = self._get_serializer(alt_path).tail(filename, self.max_lines)\n if output:\n return output\n return output\n\n @toggle_flag(\"include_stdout\")\n def get_stdout(self) -> list[str]:\n if isinstance(self._obj, KubernetesActionRun):\n # it's possible that we have a job that logs to the samestream as another job on a\n # different master (e.g., 1 job in pnw-devc and another in norcal-devc), so we\n # additionally filter by the cluster in each log message.\n # we get this information from the last attempt for this ActionRun, but\n # all of the attempts should always have the same value. This value is guaranteed\n # to be here as it's part of the PaaSTA Contract, but there's also a fallback in\n # read_log_stream_for_action_run() to use the current superregion for the tron\n # master should something go horribly wrong\n paasta_cluster = None\n if self._obj.attempts:\n paasta_cluster = self._obj.attempts[-1].command_config.env.get(\"PAASTA_CLUSTER\")\n\n return read_log_stream_for_action_run(\n action_run_id=self._obj.id,\n component=\"stdout\",\n # we update the start time of an ActionRun on a retry so we can't just use\n # that start time to figure out when we should start displaying logs for.\n # instead, we use the first attempt's start time as the date from which to\n # start getting logs from and the last attempt's end time as the date at\n # which we stop getting logs from.\n # in the case of an action that completed on its initial run, there will\n # only be one attempt, but that's fine as these single attempts will still\n # have the correct information.\n # XXX: this is suboptimal if there's many days between retries\n min_date=self._obj.attempts[0].start_time if self._obj.attempts else None,\n max_date=self._obj.attempts[-1].end_time if self._obj.attempts else None,\n paasta_cluster=paasta_cluster,\n max_lines=self.max_lines,\n )\n\n filename = actioncommand.ActionCommand.STDOUT\n output = self._get_serializer().tail(filename, self.max_lines)\n if not output:\n for alt_path in self._get_alternate_output_paths():\n output = self._get_serializer(alt_path).tail(filename, self.max_lines)\n if output:\n break\n return output\n\n @toggle_flag(\"include_stderr\")\n def get_stderr(self) -> list[str]:\n if isinstance(self._obj, KubernetesActionRun):\n # it's possible that we have a job that logs to the samestream as another job on a\n # different master (e.g., 1 job in pnw-devc and another in norcal-devc), so we\n # additionally filter by the cluster in each log message.\n # we get this information from the last attempt for this ActionRun, but\n # all of the attempts should always have the same value. This value is guaranteed\n # to be here as it's part of the PaaSTA Contract, but there's also a fallback in\n # read_log_stream_for_action_run() to use the current superregion for the tron\n # master should something go horribly wrong\n paasta_cluster = None\n if self._obj.attempts:\n paasta_cluster = self._obj.attempts[-1].command_config.env.get(\"PAASTA_CLUSTER\")\n\n return read_log_stream_for_action_run(\n action_run_id=self._obj.id,\n component=\"stderr\",\n # we update the start time of an ActionRun on a retry so we can't just use\n # that start time to figure out when we should start displaying logs for.\n # instead, we use the first attempt's start time as the date from which to\n # start getting logs from and the last attempt's end time as the date at\n # which we stop getting logs from.\n # in the case of an action that completed on its initial run, there will\n # only be one attempt, but that's fine as these single attempts will still\n # have the correct information.\n # XXX: this is suboptimal if there's many days between retries\n min_date=self._obj.attempts[0].start_time if self._obj.attempts else None,\n max_date=self._obj.attempts[-1].end_time if self._obj.attempts else None,\n paasta_cluster=paasta_cluster,\n max_lines=self.max_lines,\n )\n\n filename = actioncommand.ActionCommand.STDERR\n output = self._get_serializer().tail(filename, self.max_lines)\n if not output:\n for alt_path in self._get_alternate_output_paths():\n output = self._get_serializer(alt_path).tail(filename, self.max_lines)\n if output:\n break\n return output\n\n def get_job_name(self):\n return self._obj.job_run_id.rsplit(\".\", 1)[-2]\n\n def get_run_num(self):\n return self._obj.job_run_id.split(\".\")[-1]\n\n def get_retries_delay(self):\n if self._obj.retries_delay:\n return str(self._obj.retries_delay)\n\n def get_in_delay(self):\n if self._obj.in_delay is not None:\n return self._obj.in_delay.getTime() - time.time()\n\n def get_triggered_by(self) -> str:\n remaining = set(self._obj.remaining_triggers)\n all_triggers = sorted(self._obj.rendered_triggers)\n return \", \".join(f\"{trig}{' (done)' if trig not in remaining else ''}\" for trig in all_triggers)\n\n def get_trigger_downstreams(self) -> str:\n triggers_to_emit = self._obj.triggers_to_emit()\n return \", \".join(sorted(triggers_to_emit))\n\n\nclass ActionGraphAdapter:\n def __init__(self, action_graph):\n self.action_graph = action_graph\n\n def get_repr(self):\n def build(action_name):\n action = self.action_graph[action_name]\n dependencies = self.action_graph.get_dependencies(action_name, include_triggers=True)\n return {\n \"name\": action.name,\n \"command\": action.command,\n \"dependencies\": [d.name for d in dependencies],\n }\n\n return [build(action) for action in self.action_graph.names(include_triggers=True)]\n\n\nclass ActionRunGraphAdapter:\n def __init__(self, action_run_collection):\n self.action_runs = action_run_collection\n\n def get_repr(self):\n def build(action_run):\n graph = self.action_runs.action_graph\n dependencies = graph.get_dependencies(action_run.action_name, include_triggers=True)\n return {\n \"id\": action_run.id,\n \"name\": action_run.action_name,\n \"command\": action_run.rendered_command,\n \"raw_command\": action_run.command_config.command,\n \"state\": action_run.state,\n \"start_time\": action_run.start_time,\n \"end_time\": action_run.end_time,\n \"dependencies\": [d.name for d in dependencies],\n }\n\n def build_trigger(trigger_name):\n graph = self.action_runs.action_graph\n trigger = graph[trigger_name]\n dependencies = graph.get_dependencies(trigger_name, include_triggers=True)\n return {\n \"name\": trigger.name,\n \"command\": trigger.command,\n \"dependencies\": [d.name for d in dependencies],\n \"state\": \"unknown\", # TODO: TRON-2382: why is this hardcoded and never updated? Can we update this after improving our API timings?\n }\n\n return [build(action_run) for action_run in self.action_runs] + [\n build_trigger(trigger_name) for trigger_name in self.action_runs.action_graph.all_triggers\n ]\n\n\nclass JobRunAdapter(RunAdapter):\n\n field_names = [\n \"id\",\n \"run_num\",\n \"run_time\",\n \"start_time\",\n \"end_time\",\n \"manual\",\n \"job_name\",\n ]\n translated_field_names = [\n \"state\",\n \"node\",\n \"duration\",\n \"url\",\n \"runs\",\n \"action_graph\",\n ]\n\n def __init__(\n self,\n job_run,\n include_action_runs=False,\n include_action_graph=False,\n ):\n super().__init__(job_run)\n self.include_action_runs = include_action_runs\n self.include_action_graph = include_action_graph\n\n def get_url(self):\n return f\"/jobs/{self._obj.job_name}/{self._obj.run_num}\"\n\n @toggle_flag(\"include_action_runs\")\n def get_runs(self):\n return adapt_many(ActionRunAdapter, self._obj.action_runs, self._obj)\n\n @toggle_flag(\"include_action_graph\")\n def get_action_graph(self):\n return ActionRunGraphAdapter(self._obj.action_runs).get_repr()\n\n\nclass JobAdapter(ReprAdapter):\n\n field_names = [\"status\", \"all_nodes\", \"allow_overlap\", \"queueing\"]\n translated_field_names = [\n \"name\",\n \"scheduler\",\n \"action_names\",\n \"node_pool\",\n \"last_success\",\n \"next_run\",\n \"url\",\n \"runs\",\n \"max_runtime\",\n \"action_graph\",\n \"monitoring\",\n \"expected_runtime\",\n \"actions_expected_runtime\",\n ]\n\n def __init__(\n self,\n job,\n include_job_runs=False,\n include_action_runs=False,\n include_action_graph=True,\n include_node_pool=True,\n num_runs=None,\n ):\n super().__init__(job)\n self.include_job_runs = include_job_runs\n self.include_action_runs = include_action_runs\n self.include_action_graph = include_action_graph\n self.include_node_pool = include_node_pool\n self.num_runs = num_runs\n\n def get_name(self):\n return self._obj.get_name()\n\n def get_monitoring(self):\n return self._obj.get_monitoring()\n\n def get_scheduler(self):\n return SchedulerAdapter(self._obj.scheduler).get_repr()\n\n def get_action_names(self):\n return list(self._obj.action_graph.names())\n\n @toggle_flag(\"include_node_pool\")\n def get_node_pool(self):\n return NodePoolAdapter(self._obj.node_pool).get_repr()\n\n def get_last_success(self):\n last_success = self._obj.runs.last_success\n return last_success.end_time if last_success else None\n\n def get_next_run(self):\n next_run = self._obj.runs.next_run\n return next_run.run_time if next_run else None\n\n def get_url(self):\n return f\"/jobs/{quote(self._obj.get_name())}\"\n\n @toggle_flag(\"include_job_runs\")\n def get_runs(self):\n runs = adapt_many(\n JobRunAdapter,\n list(self._obj.runs)[: self.num_runs or None],\n self.include_action_runs,\n )\n return runs\n\n def get_max_runtime(self):\n return str(self._obj.max_runtime)\n\n def get_expected_runtime(self):\n return delta_total_seconds(self._obj.expected_runtime)\n\n def get_actions_expected_runtime(self):\n return self._obj.action_graph.expected_runtime\n\n @toggle_flag(\"include_action_graph\")\n def get_action_graph(self):\n return ActionGraphAdapter(self._obj.action_graph).get_repr()\n\n\nclass JobIndexAdapter(ReprAdapter):\n\n translated_field_names = [\"name\", \"actions\"]\n\n def get_name(self):\n return self._obj.get_name()\n\n def get_actions(self):\n def adapt_run(run):\n return {\"name\": run.action_name, \"command\": run.command_config.command}\n\n job_run = self._obj.get_runs().get_newest()\n if not job_run:\n return []\n return [adapt_run(action_run) for action_run in job_run.action_runs]\n\n\nclass SchedulerAdapter(ReprAdapter):\n\n translated_field_names = [\"value\", \"type\", \"jitter\"]\n\n def get_value(self):\n return self._obj.get_value()\n\n def get_type(self):\n return self._obj.get_name()\n\n def get_jitter(self):\n return scheduler.get_jitter_str(self._obj.get_jitter())\n\n\nclass EventAdapter(ReprAdapter):\n\n field_names = [\"name\", \"entity\", \"time\"]\n translated_field_names = [\"level\"]\n\n def get_level(self):\n return self._obj.level.label\n\n\nclass NodeAdapter(ReprAdapter):\n field_names = [\"name\", \"hostname\", \"username\", \"port\"]\n\n\nclass NodePoolAdapter(ReprAdapter):\n translated_field_names = [\"name\", \"nodes\"]\n\n def get_name(self):\n return self._obj.get_name()\n\n def get_nodes(self):\n return adapt_many(NodeAdapter, self._obj.get_nodes())\n" }, { "path": "tron/api/async_resource.py", "content": "import threading\nimport time\n\nfrom twisted.internet import threads\nfrom twisted.web import server\n\nfrom tron.metrics import timer\n\n\ndef report_resource_request(resource, request, duration_ms):\n timer(\n name=f\"tron.api.{resource.__class__.__name__}\",\n delta=duration_ms,\n dimensions={\"method\": request.method.decode()},\n )\n\n\nclass AsyncResource:\n capacity = 10\n semaphore = threading.Semaphore(value=capacity)\n lock = threading.Lock()\n\n @staticmethod\n def finish(result, request, resource):\n result, duration_ms = result\n request.write(result)\n request.finish()\n report_resource_request(resource, request, duration_ms)\n\n @staticmethod\n def process(fn, resource, request):\n start = time.time()\n with AsyncResource.semaphore:\n result = fn(resource, request)\n duration_ms = 1000 * (time.time() - start)\n return result, duration_ms\n\n @staticmethod\n def bounded(fn):\n def wrapper(resource, request):\n d = threads.deferToThread(\n AsyncResource.process,\n fn,\n resource,\n request,\n )\n d.addCallback(AsyncResource.finish, request, resource)\n d.addErrback(request.processingFailed)\n return server.NOT_DONE_YET\n\n return wrapper\n\n @staticmethod\n def exclusive(fn):\n def wrapper(resource, request):\n # ensures only one exclusive request starts consuming the semaphore\n start = time.time()\n with AsyncResource.lock:\n # this will wait until all bounded requests finished processing\n for _ in range(AsyncResource.capacity):\n AsyncResource.semaphore.acquire()\n try:\n return fn(resource, request)\n finally:\n for _ in range(AsyncResource.capacity):\n AsyncResource.semaphore.release()\n duration_ms = 1000 * (time.time() - start)\n report_resource_request(resource, request, duration_ms)\n\n return wrapper\n" }, { "path": "tron/api/auth.py", "content": "import logging\nimport os\nimport re\nfrom functools import lru_cache\nfrom typing import NamedTuple\n\nimport cachetools.func\nimport requests\nfrom twisted.web.server import Request\n\n\nlogger = logging.getLogger(__name__)\nAUTH_CACHE_SIZE = 50000\nAUTH_CACHE_TTL = 30 * 60\nSERVICE_NAME_PATH_PATTERN = re.compile(r\"^/api/jobs/([^/.]+)\")\n\n\nclass AuthorizationOutcome(NamedTuple):\n authorized: bool\n reason: str\n\n\nclass AuthorizationFilter:\n \"\"\"API request authorization via external system\"\"\"\n\n def __init__(self, endpoint: str, enforce: bool):\n \"\"\"Constructor\n\n :param str endpoint: HTTP endpoint of external authorization system\n :param bool enforce: whether to enforce authorization decisions\n \"\"\"\n self.endpoint = endpoint\n self.enforce = enforce\n self.session = requests.Session()\n\n @classmethod\n @lru_cache(maxsize=1)\n def get_from_env(cls) -> \"AuthorizationFilter\":\n return cls(\n endpoint=os.getenv(\"API_AUTH_ENDPOINT\", \"\"),\n enforce=bool(os.getenv(\"API_AUTH_ENFORCE\", \"\")),\n )\n\n def is_request_authorized(self, request: Request) -> AuthorizationOutcome:\n \"\"\"Check if API request is authorized\n\n :param Request request: API request object\n :return: auth outcome\n \"\"\"\n if not self.endpoint:\n return AuthorizationOutcome(True, \"Auth not enabled\")\n token = (request.getHeader(\"Authorization\") or \"\").strip()\n token = token.split()[-1] if token else \"\" # removes \"Bearer\" prefix\n url_path = request.path.decode() if request.path is not None else \"\" # type: ignore[attr-defined] # mypy does not like what twisted is doing here\n auth_outcome = self._is_request_authorized_impl(\n # path and method are byte arrays in twisted\n path=url_path,\n token=token,\n method=request.method.decode(),\n service=self._extract_service_from_path(url_path),\n )\n return auth_outcome if self.enforce else AuthorizationOutcome(True, \"Auth dry-run\")\n\n @cachetools.func.ttl_cache(maxsize=AUTH_CACHE_SIZE, ttl=AUTH_CACHE_TTL)\n def _is_request_authorized_impl(\n self,\n path: str,\n token: str,\n method: str,\n service: str | None,\n ) -> AuthorizationOutcome:\n \"\"\"Check if API request is authorized\n\n :param str path: API path\n :param str token: authentication token\n :param str method: http method\n :return: auth outcome\n \"\"\"\n try:\n response = self.session.post(\n url=self.endpoint,\n json={\n \"input\": {\n \"path\": path,\n \"backend\": \"tron\",\n \"token\": token,\n \"method\": method.lower(),\n \"service\": service,\n },\n },\n timeout=2,\n ).json()\n except Exception as e:\n logger.exception(f\"Issue communicating with auth endpoint: {e}\")\n return AuthorizationOutcome(False, \"Auth backend error\")\n\n auth_result_allowed = response.get(\"result\", {}).get(\"allowed\")\n if auth_result_allowed is None:\n return AuthorizationOutcome(False, \"Malformed auth response\")\n\n if not auth_result_allowed:\n reason = response[\"result\"].get(\"reason\", \"Denied\")\n return AuthorizationOutcome(False, reason)\n\n reason = response[\"result\"].get(\"reason\", \"Ok\")\n return AuthorizationOutcome(True, reason)\n\n @staticmethod\n def _extract_service_from_path(path: str) -> str | None:\n \"\"\"If a request path contains a service name, extract it.\n\n Example:\n /api/jobs/someservice.instance/110/run -> someservice\n\n :param str path: request path\n :return: service name, or None if not found\n \"\"\"\n match = SERVICE_NAME_PATH_PATTERN.search(path)\n return match.group(1) if match else None\n" }, { "path": "tron/api/controller.py", "content": "\"\"\"\nWeb Controllers for the API.\n\"\"\"\nimport logging\nfrom typing import TYPE_CHECKING\nfrom typing import TypedDict\n\nfrom tron import yaml\nfrom tron.config.manager import ConfigManager\nfrom tron.core.actionrun import ActionRun\nfrom tron.core.jobrun import JobRun\nfrom tron.eventbus import EventBus\n\nif TYPE_CHECKING:\n from tron.mcp import MasterControlProgram\n\nlog = logging.getLogger(__name__)\n\n\nclass UnknownCommandError(Exception):\n \"\"\"Exception raised when a controller received an unknown command.\"\"\"\n\n\nclass InvalidCommandForActionState(Exception):\n \"\"\"\n Exception raised when a controller attempts a command on an action in a state\n that does not support that command (e.g., skipping a successful run).\n \"\"\"\n\n def __init__(self, command: str, action_name: str, action_state: str) -> None:\n self.command = command\n self.action_name = action_name\n self.action_state = action_state\n self.message = f\"Failed to {command} on {action_name}. State is {action_state}.\"\n super().__init__()\n\n\nclass JobCollectionController:\n def __init__(self, job_collection):\n self.job_collection = job_collection\n\n def handle_command(self, command, old_name=None, new_name=None):\n if command == \"move\":\n if old_name not in self.job_collection.get_names():\n return f\"Error: {old_name} doesn't exist\"\n if new_name in self.job_collection.get_names():\n return f\"Error: {new_name} exists already\"\n return self.job_collection.move(old_name, new_name)\n\n raise UnknownCommandError(f\"Unknown command {command}. Try running this on an individual job or action run id\")\n\n\nclass ActionRunController:\n\n mapped_commands = {\n \"start\",\n \"success\",\n \"cancel\",\n \"fail\",\n \"skip\",\n \"stop\",\n \"kill\",\n \"retry\",\n \"recover\",\n }\n\n def __init__(self, action_run: ActionRun, job_run: JobRun) -> None:\n self.action_run = action_run\n self.job_run = job_run\n\n def handle_command(self, command, **kwargs):\n if command not in self.mapped_commands:\n raise UnknownCommandError(\n f\"Unknown command {command}. You can only do one of the following to Action runs: {self.mapped_commands}\"\n )\n\n if command == \"start\" and self.job_run.is_scheduled:\n return \"Action run cannot be started if its job run is still \" \"scheduled.\"\n\n if command == \"recover\" and not self.action_run.is_unknown:\n return \"Action run cannot be recovered if its state is not unknown.\"\n\n if command in (\"stop\", \"kill\"):\n return self.handle_termination(command)\n\n if command == \"retry\":\n original_command = not kwargs.get(\"use_latest_command\", False)\n return self.handle_retry(original_command)\n\n if getattr(self.action_run, command)():\n msg = \"%s now in state %s\"\n return msg % (self.action_run, self.action_run.state)\n\n raise InvalidCommandForActionState(\n command=command,\n action_name=self.action_run.name,\n action_state=self.action_run.state,\n )\n\n def handle_termination(self, command):\n try:\n # Extra message is only used for killing mesos action as warning so far.\n extra_msg = getattr(self.action_run, command)()\n msg = \"Attempting to %s %s\"\n if extra_msg is not None:\n msg = msg + \"\\n\" + extra_msg\n return msg % (command, self.action_run)\n except NotImplementedError as e:\n msg = \"Failed to %s: %s\"\n return msg % (command, e)\n\n def handle_retry(self, original_command):\n cleanup_run = self.job_run.action_runs.cleanup_action_run\n if cleanup_run and cleanup_run.is_done:\n return \"JobRun has run a cleanup action, use rerun instead\"\n\n if self.action_run.retry(original_command=original_command):\n return \"Retrying %s\" % self.action_run\n else:\n return \"Failed to schedule retry for %s\" % self.action_run\n\n\nclass JobRunController:\n\n mapped_commands = {\"start\", \"success\", \"cancel\", \"fail\", \"stop\"}\n\n def __init__(self, job_run, job_scheduler):\n self.job_run = job_run\n self.job_scheduler = job_scheduler\n\n def handle_command(self, command):\n # as of TRON-1340, `tronctl backfill` depends on this response value\n # (i.e. \"Created JobRun:\"). be careful when changing this!\n if command == \"restart\" or command == \"rerun\":\n runs = self.job_scheduler.manual_start(self.job_run.run_time)\n return \"Created %s\" % \",\".join(str(run) for run in runs)\n\n if command in self.mapped_commands:\n if getattr(self.job_run, command)():\n return f\"{self.job_run} now in state {self.job_run.state}\"\n\n msg = \"Failed to %s, %s in state %s\"\n return msg % (command, self.job_run, self.job_run.state)\n\n if command == \"retry\":\n raise UnknownCommandError(\n \"Error: Job runs cannot be retried, only individual actions can. Did you mean 'rerun'?\"\n )\n else:\n raise UnknownCommandError(\n f\"Unknown command {command}. Only one of the following applies to a Job run: {self.mapped_commands}\"\n )\n\n\nclass JobController:\n def __init__(self, job_scheduler):\n self.job_scheduler = job_scheduler\n\n def handle_command(self, command, run_time=None):\n if command == \"enable\":\n self.job_scheduler.enable()\n return \"%s is enabled\" % self.job_scheduler.get_job()\n\n elif command == \"disable\":\n self.job_scheduler.disable()\n return \"%s is disabled\" % self.job_scheduler.get_job()\n\n elif command == \"start\":\n runs = self.job_scheduler.manual_start(run_time=run_time)\n return \"Created %s\" % \",\".join(str(run) for run in runs)\n\n if command == \"retry\":\n raise UnknownCommandError(\n \"Error: A whole Job cannot be retried, only individual actions for a specific job run id can.\"\n )\n elif command in [\"stop\", \"success\", \"cancel\", \"fail\", \"stop\"]:\n raise UnknownCommandError(\n f\"Error: {command} doesn't apply to a whole Job. Please run this on an individual job run id. Hint: try '{self.job_scheduler.get_job()}.-1' for the latest job id\"\n )\n else:\n raise UnknownCommandError(\n f\"Unknown command {command}. Does it apply to a whole job? Try a specific Job id or individual action\"\n )\n\n\nclass ConfigResponse(TypedDict):\n config: str\n hash: str\n\n\nclass ConfigController:\n \"\"\"Control config. Return config contents and accept updated configuration\n from the API.\n \"\"\"\n\n DEFAULT_NAMED_CONFIG = \"\\njobs:\\n\"\n\n def __init__(self, mcp: \"MasterControlProgram\") -> None:\n self.mcp = mcp\n self.config_manager: ConfigManager = mcp.get_config_manager()\n\n def _get_config_content(self, name: str) -> str:\n if name not in self.config_manager:\n return self.DEFAULT_NAMED_CONFIG\n return self.config_manager.read_raw_config(name)\n\n def read_config(self, name: str) -> ConfigResponse:\n config_content = self._get_config_content(name)\n config_hash = self.config_manager.get_hash(name)\n return {\"config\": config_content, \"hash\": config_hash}\n\n def read_all_configs(self) -> dict[str, ConfigResponse]:\n configs = {}\n\n for service in self.config_manager.get_namespaces():\n config: ConfigResponse = {\n \"config\": self._get_config_content(service),\n \"hash\": self.config_manager.get_hash(service),\n }\n configs[service] = config\n\n return configs\n\n def check_config(self, name, content, config_hash):\n \"\"\"Update a configuration fragment and reload the MCP.\"\"\"\n if self.config_manager.get_hash(name) != config_hash:\n return \"Configuration update will fail: config is stale, try again\"\n\n try:\n content = yaml.load(content)\n self.config_manager.validate_with_fragment(name, content)\n except Exception as e:\n return \"Configuration update will fail: %s\" % str(e)\n\n def update_config(self, name, content, config_hash):\n \"\"\"Update a configuration fragment and reload the MCP.\"\"\"\n if self.config_manager.get_hash(name) != config_hash:\n return \"Configuration has changed. Please try again.\"\n\n old_config = self.read_config(name)[\"config\"]\n try:\n log.info(f\"Reconfiguring namespace {name}\")\n self.config_manager.write_config(name, content)\n self.mcp.reconfigure(namespace=name)\n except Exception as e:\n log.error(f\"Configuration for {name} update failed: {e}\")\n log.error(\"Reconfiguring with the previous good configuration\")\n try:\n self.config_manager.write_config(name, old_config)\n self.mcp.reconfigure(namespace=name)\n except Exception as e:\n log.error(\"Could not restore old config: %s\" % e)\n return str(e)\n return str(e)\n\n def delete_config(self, name, content, config_hash):\n \"\"\"Delete a configuration fragment and reload the MCP.\"\"\"\n if self.config_manager.get_hash(name) != config_hash:\n return \"Configuration has changed. Please try again.\"\n\n if content != \"\":\n return \"Configuration content is not empty, will not delete.\"\n\n try:\n log.info(f\"Deleting namespace {name}\")\n self.config_manager.delete_config(name)\n self.mcp.reconfigure(namespace=name)\n except Exception as e:\n log.error(f\"Deleting configuration for {name} failed: {e}\")\n return str(e)\n\n def get_namespaces(self):\n return self.config_manager.get_namespaces()\n\n\nclass EventsController:\n COMMANDS = {\"publish\", \"discard\"}\n\n def publish(self, event):\n if not EventBus.instance:\n return dict(error=\"EventBus disabled\")\n\n if EventBus.has_event(event):\n msg = f\"event {event} already published\"\n log.warning(msg)\n return dict(response=msg)\n\n if not EventBus.publish(event):\n msg = f\"could not publish {event}\"\n log.error(msg)\n return dict(error=msg)\n\n return dict(response=\"OK\")\n\n def discard(self, event):\n if not EventBus.instance:\n return dict(error=\"EventBus disabled\")\n\n if not EventBus.discard(event):\n msg = f\"could not discard {event}\"\n log.error(msg)\n return dict(error=msg)\n\n return dict(response=\"OK\")\n\n def info(self):\n if not EventBus.instance:\n return dict(error=\"EventBus disabled\")\n\n return dict(response=EventBus.instance.event_log)\n" }, { "path": "tron/api/requestargs.py", "content": "\"\"\"Functions for returning validated values from a twisted.web.Request object.\n\"\"\"\nimport datetime\n\nDATE_FORMAT = \"%Y-%m-%d %H:%M:%S\"\n\n\ndef get_integer(request, key):\n \"\"\"Returns the first value in the request args for the given key, if that\n value is an integer. Otherwise returns None.\n \"\"\"\n value = get_string(request, key)\n if value is None or not value.isdigit():\n return None\n\n return int(value)\n\n\ndef get_string(request, key):\n \"\"\"Returns the first value in the request args for a given key.\"\"\"\n if not request.args:\n return None\n\n if type(key) is not bytes:\n key = key.encode()\n\n if key not in request.args:\n return None\n\n val = request.args[key][0]\n if val is not None and type(val) is bytes:\n val = val.decode()\n\n return val\n\n\ndef get_bool(request, key, default=None):\n \"\"\"Returns True if the key exists and is truthy in the request args.\"\"\"\n int_value = get_integer(request, key)\n if int_value is None:\n return default\n\n return bool(int_value)\n\n\ndef get_datetime(request, key):\n \"\"\"Returns the first value in the request args for a given key. Casts to\n a datetime. Returns None if the value cannot be converted to datetime.\n \"\"\"\n val = get_string(request, key)\n if not val:\n return None\n\n try:\n return datetime.datetime.strptime(val, DATE_FORMAT)\n except ValueError:\n return None\n" }, { "path": "tron/api/resource.py", "content": "\"\"\"\nWeb Services Interface used by command-line clients and web frontend to\nview current state, event history and send commands to trond.\n\"\"\"\nimport collections\nimport datetime\nimport json\nimport logging\nimport traceback\n\nimport staticconf\nfrom prometheus_client.twisted import MetricsResource as MetricsResourceProm\nfrom twisted.web import http\nfrom twisted.web import resource\nfrom twisted.web import server\nfrom twisted.web import static\n\nfrom tron import __version__\nfrom tron.api import adapter\nfrom tron.api import controller\nfrom tron.api import requestargs\nfrom tron.api.async_resource import AsyncResource\nfrom tron.api.auth import AuthorizationFilter\nfrom tron.config.static_config import get_config_watcher\nfrom tron.config.static_config import NAMESPACE\nfrom tron.metrics import meter\nfrom tron.metrics import view_all_metrics\nfrom tron.utils import maybe_decode\n\nlog = logging.getLogger(__name__)\n\n\nclass JSONEncoder(json.JSONEncoder):\n \"\"\"Custom JSON for certain objects\"\"\"\n\n def default(self, o):\n if isinstance(o, datetime.datetime):\n return o.strftime(\"%Y-%m-%d %H:%M:%S\")\n\n if isinstance(o, datetime.date):\n return o.isoformat()\n\n if isinstance(o, collections.abc.KeysView):\n return list(o)\n\n return super().default(o)\n\n\ndef respond(request, response, code=None, headers=None):\n \"\"\"Helper to generate a json response\"\"\"\n if code is None:\n if type(response) is dict and response.get(\"error\"):\n code = http.INTERNAL_SERVER_ERROR\n else:\n code = http.OK\n request.setResponseCode(code)\n request.setHeader(b\"content-type\", b\"application/json; charset=utf-8\")\n request.setHeader(b\"Access-Control-Allow-Origin\", b\"*\")\n for key, val in (headers or {}).items():\n request.setHeader(str(key), str(val))\n\n result = (\n json.dumps(\n response,\n cls=JSONEncoder,\n )\n if response\n else \"\"\n )\n\n if type(result) is not bytes:\n result = result.encode(\"utf8\")\n\n return result\n\n\ndef handle_command(request, api_controller, obj, **kwargs):\n \"\"\"Handle a request to perform a command.\"\"\"\n command = requestargs.get_string(request, \"command\")\n log.info(\"Handling '%s' request on %s\", command, obj)\n try:\n response = api_controller.handle_command(command, **kwargs)\n return respond(request=request, response={\"result\": response})\n except controller.UnknownCommandError:\n error_msg = f\"Unknown command '{command}' for '{obj}'\"\n log.warning(error_msg)\n return respond(\n request=request,\n response={\"error\": error_msg},\n code=http.NOT_IMPLEMENTED,\n )\n except controller.InvalidCommandForActionState as e:\n log.warning(e.message)\n return respond(\n request=request,\n response={\"error\": e.message},\n code=http.CONFLICT,\n )\n except Exception as e:\n log.exception(\"%r while executing command %s for %s\", e, command, obj)\n trace = traceback.format_exc()\n return respond(request=request, response={\"error\": trace})\n\n\nclass AuthenticatedResource(resource.Resource):\n def render(self, request):\n \"\"\"Overriding base `render` method to support authentication\"\"\"\n auth_outcome = AuthorizationFilter.get_from_env().is_request_authorized(request)\n if not auth_outcome.authorized:\n return respond(\n request=request,\n response={\"error\": f\"Auth failed (reason: {auth_outcome.reason})\"},\n code=http.FORBIDDEN,\n headers={\"X-Auth-Failure-Reason\": auth_outcome.reason},\n )\n return super().render(request)\n\n\nclass ErrorResource(resource.Resource):\n \"\"\"Equivalent to resource.NoResource, except error message is returned\n as JSON, not HTML\"\"\"\n\n def __init__(self, error=\"No Such Resource\", code=http.NOT_FOUND):\n resource.Resource.__init__(self)\n self.code = code\n self.error = error\n\n @AsyncResource.bounded\n def render_GET(self, request):\n return respond(request=request, response={\"error\": self.error}, code=self.code)\n\n @AsyncResource.exclusive\n def render_POST(self, request):\n return respond(request=request, response={\"error\": self.error}, code=self.code)\n\n def getChild(self, chnam, request):\n \"\"\"Overrided getChild to ensure a NoResource is not returned\"\"\"\n return self\n\n\ndef resource_from_collection(collection, name, child_resource):\n \"\"\"Return a child resource from a collection by name. If no item is found,\n return ErrorResource.\n \"\"\"\n item = collection.get_by_name(name)\n if item is None:\n return ErrorResource(\"Cannot find child '%s'\" % name)\n return child_resource(item)\n\n\nclass ActionRunResource(AuthenticatedResource):\n\n isLeaf = True\n\n def __init__(self, action_run, job_run):\n resource.Resource.__init__(self)\n self.action_run = action_run\n self.job_run = job_run\n self.controller = controller.ActionRunController(action_run, job_run)\n self.config_watcher = get_config_watcher()\n\n @AsyncResource.bounded\n def render_GET(self, request):\n run_adapter = adapter.ActionRunAdapter(\n self.action_run,\n self.job_run,\n requestargs.get_integer(request, \"num_lines\")\n or staticconf.read(\"logging.max_lines_to_display\", namespace=NAMESPACE),\n include_stdout=requestargs.get_bool(request, \"include_stdout\"),\n include_stderr=requestargs.get_bool(request, \"include_stderr\"),\n include_meta=requestargs.get_bool(request, \"include_meta\"),\n )\n return respond(request=request, response=run_adapter.get_repr())\n\n @AsyncResource.exclusive\n def render_POST(self, request):\n use_latest_command = requestargs.get_bool(request, \"use_latest_command\", False)\n return handle_command(\n request,\n self.controller,\n self.action_run,\n use_latest_command=use_latest_command,\n )\n\n\nclass JobRunResource(AuthenticatedResource):\n def __init__(self, job_run, job_scheduler):\n resource.Resource.__init__(self)\n self.job_run = job_run\n self.job_scheduler = job_scheduler\n self.controller = controller.JobRunController(job_run, job_scheduler)\n\n def getChild(self, action_name, _):\n if not action_name:\n return self\n\n action_name = maybe_decode(\n action_name\n ) # TODO: TRON-2293 maybe_decode is a relic of Python2->Python3 migration. Remove it.\n if action_name in self.job_run.action_runs:\n action_run = self.job_run.action_runs[action_name]\n return ActionRunResource(action_run, self.job_run)\n\n return ErrorResource(\n f\"Cannot find action '{action_name}' for \" f\"'{self.job_run}'\",\n )\n\n @AsyncResource.bounded\n def render_GET(self, request):\n include_runs = requestargs.get_bool(request, \"include_action_runs\")\n include_graph = requestargs.get_bool(request, \"include_action_graph\")\n run_adapter = adapter.JobRunAdapter(\n self.job_run,\n include_action_runs=include_runs,\n include_action_graph=include_graph,\n )\n return respond(request=request, response=run_adapter.get_repr())\n\n @AsyncResource.exclusive\n def render_POST(self, request):\n return handle_command(request, self.controller, self.job_run)\n\n\ndef is_negative_int(string):\n return string.startswith(\"-\") and string[1:].isdigit()\n\n\nclass JobResource(AuthenticatedResource):\n def __init__(self, job_scheduler):\n resource.Resource.__init__(self)\n self.job_scheduler = job_scheduler\n self.controller = controller.JobController(job_scheduler)\n\n def get_run_from_identifier(self, run_id):\n job_runs = self.job_scheduler.get_job_runs()\n if run_id.upper() == \"HEAD\":\n return job_runs.get_newest()\n if run_id.isdigit():\n return job_runs.get_run_by_num(int(run_id))\n if is_negative_int(run_id):\n return job_runs.get_run_by_index(int(run_id))\n\n def getChild(self, run_id, _):\n if not run_id:\n return self\n\n run_id = maybe_decode(\n run_id\n ) # TODO: TRON-2293 maybe_decode is a relic of Python2->Python3 migration. Remove it.\n run = self.get_run_from_identifier(run_id)\n if run:\n return JobRunResource(run, self.job_scheduler)\n\n job = self.job_scheduler.get_job()\n if run_id in job.action_graph.names():\n action_runs = job.runs.get_action_runs(run_id)\n return ActionRunHistoryResource(action_runs)\n\n return ErrorResource(f\"Cannot find job run '{run_id}' for '{job}'\")\n\n @AsyncResource.bounded\n def render_GET(self, request):\n include_action_runs = requestargs.get_bool(\n request,\n \"include_action_runs\",\n )\n include_graph = requestargs.get_bool(request, \"include_action_graph\")\n num_runs = requestargs.get_integer(request, \"num_runs\")\n job_adapter = adapter.JobAdapter(\n self.job_scheduler.get_job(),\n include_job_runs=True,\n include_action_runs=include_action_runs,\n include_action_graph=include_graph,\n num_runs=num_runs,\n )\n return respond(request=request, response=job_adapter.get_repr())\n\n @AsyncResource.exclusive\n def render_POST(self, request):\n run_time = requestargs.get_datetime(request, \"run_time\")\n return handle_command(\n request,\n self.controller,\n self.job_scheduler,\n run_time=run_time,\n )\n\n\nclass ActionRunHistoryResource(AuthenticatedResource):\n\n isLeaf = True\n\n def __init__(self, action_runs):\n resource.Resource.__init__(self)\n self.action_runs = action_runs\n\n @AsyncResource.bounded\n def render_GET(self, request):\n return respond(\n request=request,\n response=adapter.adapt_many(adapter.ActionRunAdapter, self.action_runs),\n )\n\n\nclass JobCollectionResource(AuthenticatedResource):\n def __init__(self, job_collection):\n self.job_collection = job_collection\n self.controller = controller.JobCollectionController(job_collection)\n resource.Resource.__init__(self)\n\n def getChild(self, name, request):\n if not name:\n return self\n\n name = maybe_decode(name) # TODO: TRON-2293 maybe_decode is a relic of Python2->Python3 migration. Remove it.\n return resource_from_collection(self.job_collection, name, JobResource)\n\n def get_data(\n self,\n include_job_run=False,\n include_action_runs=False,\n include_action_graph=True,\n include_node_pool=True,\n ):\n return adapter.adapt_many(\n adapter.JobAdapter,\n self.job_collection.get_jobs(),\n include_job_run,\n include_action_runs,\n include_action_graph,\n include_node_pool,\n num_runs=5,\n )\n\n def get_job_index(self):\n jobs = adapter.adapt_many(\n adapter.JobIndexAdapter,\n self.job_collection.get_jobs(),\n )\n return {job[\"name\"]: job[\"actions\"] for job in jobs}\n\n @AsyncResource.bounded\n def render_GET(self, request):\n include_job_runs = requestargs.get_bool(\n request,\n \"include_job_runs\",\n default=False,\n )\n include_action_runs = requestargs.get_bool(\n request,\n \"include_action_runs\",\n default=False,\n )\n include_action_graph = requestargs.get_bool(\n request,\n \"include_action_graph\",\n default=True,\n )\n include_node_pool = requestargs.get_bool(\n request,\n \"include_node_pool\",\n default=True,\n )\n response = dict(\n jobs=self.get_data(\n include_job_runs,\n include_action_runs,\n include_action_graph,\n include_node_pool,\n ),\n )\n return respond(request=request, response=response)\n\n @AsyncResource.exclusive\n def render_POST(self, request):\n old_name = requestargs.get_string(request, \"old_name\")\n new_name = requestargs.get_string(request, \"new_name\")\n return handle_command(\n request=request,\n api_controller=self.controller,\n obj=self.job_collection,\n old_name=old_name,\n new_name=new_name,\n )\n\n\nclass ConfigResource(AuthenticatedResource):\n \"\"\"Resource for configuration changes\"\"\"\n\n isLeaf = True\n\n def __init__(self, master_control):\n self.controller = controller.ConfigController(master_control)\n resource.Resource.__init__(self)\n\n def get_config_index(self):\n return self.controller.get_namespaces()\n\n @AsyncResource.bounded\n def render_GET(self, request):\n config_name = requestargs.get_string(request, \"name\")\n if not config_name:\n response = self.controller.read_all_configs()\n else:\n response = self.controller.read_config(config_name)\n return respond(request=request, response=response)\n\n @AsyncResource.exclusive\n def render_POST(self, request):\n config_content = requestargs.get_string(request, \"config\")\n name = requestargs.get_string(request, \"name\")\n config_hash = requestargs.get_string(request, \"hash\")\n check = requestargs.get_bool(request, \"check\")\n\n if not name:\n return respond(\n request=request,\n response={\"error\": \"'name' for config is required.\"},\n code=http.BAD_REQUEST,\n )\n\n response = {\"status\": \"Active\"}\n\n if check:\n fn = self.controller.check_config\n req = \"configure check\"\n elif config_content == \"\":\n fn = self.controller.delete_config\n req = \"configuration delete\"\n else:\n fn = self.controller.update_config\n req = \"reconfigure\"\n\n log.info(f\"Handling {req} request: {name}, {config_hash}\")\n error = fn(name, config_content, config_hash)\n\n if error:\n response[\"error\"] = error\n return respond(request=request, response=response)\n\n\nclass StatusResource(resource.Resource):\n\n isLeaf = True\n\n def __init__(self, master_control):\n self._master_control = master_control\n resource.Resource.__init__(self)\n\n @AsyncResource.bounded\n def render_GET(self, request):\n return respond(\n request=request,\n response={\n \"status\": \"I'm alive.\",\n \"version\": __version__,\n \"boot_time\": int(self._master_control.boot_time),\n },\n )\n\n\nclass MetricsResource(resource.Resource):\n\n isLeaf = True\n\n def __init__(self):\n resource.Resource.__init__(self)\n\n @AsyncResource.exclusive\n def render_GET(self, request):\n return respond(request=request, response=view_all_metrics())\n\n\nclass EventsResource(AuthenticatedResource):\n isLeaf = True\n\n def __init__(self):\n super().__init__()\n self.controller = controller.EventsController()\n\n @AsyncResource.exclusive\n def render_GET(self, request):\n response = self.controller.info()\n return respond(request=request, response=response)\n\n @AsyncResource.bounded\n def render_POST(self, request):\n command = requestargs.get_string(request, \"command\")\n if command not in self.controller.COMMANDS:\n return respond(\n request=request,\n response=dict(error=f\"Unknown command: {command}\"),\n code=http.BAD_REQUEST,\n )\n event = requestargs.get_string(request, \"event\")\n fn = getattr(self.controller, command)\n response = fn(event)\n return respond(request=request, response=response)\n\n\nclass ApiRootResource(AuthenticatedResource):\n def __init__(self, mcp):\n self._master_control = mcp\n resource.Resource.__init__(self)\n\n # Setup children\n self.putChild(\n b\"jobs\",\n JobCollectionResource(mcp.get_job_collection()),\n )\n\n self.putChild(b\"config\", ConfigResource(mcp))\n self.putChild(b\"status\", StatusResource(mcp))\n self.putChild(b\"events\", EventsResource())\n self.putChild(b\"metrics\", MetricsResource())\n self.putChild(b\"prom-metrics\", MetricsResourceProm())\n self.putChild(b\"\", self)\n\n @AsyncResource.bounded\n def render_GET(self, request):\n \"\"\"Return an index of urls for resources.\"\"\"\n response = {\n \"jobs\": self.children[b\"jobs\"].get_job_index(),\n \"namespaces\": self.children[b\"config\"].get_config_index(),\n }\n return respond(request=request, response=response)\n\n\nclass RootResource(resource.Resource):\n def __init__(self, mcp, web_path):\n resource.Resource.__init__(self)\n self.web_path = web_path\n self.mcp = mcp\n self.putChild(b\"api\", ApiRootResource(self.mcp))\n self.putChild(b\"web\", static.File(web_path))\n self.putChild(b\"\", self)\n\n def render_GET(self, request):\n request.redirect(b\"/web\")\n request.finish()\n return server.NOT_DONE_YET\n\n def __str__(self):\n return f\"{type(self).__name__}({self.mcp}, {self.web_path})\"\n\n\nclass LogAdapter:\n def __init__(self, logger):\n self.logger = logger\n\n def write(self, line):\n self.logger.info(line.rstrip(b\"\\n\"))\n\n def close(self):\n pass\n\n\nclass TronSite(server.Site):\n \"\"\"Subclass of a twisted Site to customize logging.\"\"\"\n\n access_log = logging.getLogger(\"tron.api.www.access\")\n\n @classmethod\n def create(cls, mcp, web_path):\n return cls(RootResource(mcp, web_path))\n\n def startFactory(self):\n server.Site.startFactory(self)\n self.logFile = LogAdapter(self.access_log)\n\n def log(self, request):\n super().log(request)\n if 200 <= request.code < 300:\n meter(\"tron.site.2xx\")\n if 300 <= request.code < 400:\n meter(\"tron.site.3xx\")\n if 400 <= request.code < 500:\n meter(\"tron.site.4xx\")\n if 500 <= request.code < 600:\n meter(\"tron.site.5xx\")\n\n def __repr__(self):\n return f\"{self.__class__.__name__}({self.resource})\"\n" }, { "path": "tron/bin/action_runner.py", "content": "#!/usr/bin/env python3.10\n\"\"\"\nWrite pid and stdout/stderr to a standard location before execing a command.\n\"\"\"\nimport argparse\nimport contextlib\nimport logging\nimport os\nimport subprocess\nimport sys\nimport threading\nimport time\n\nfrom tron import yaml\n\nSTATUS_FILE = \"status\"\n\n\nclass StatusFile:\n \"\"\"Manage a status file.\"\"\"\n\n def __init__(self, filename):\n self.filename = filename\n\n def get_content(self, run_id, command, proc):\n return {\n \"run_id\": run_id,\n \"command\": command,\n \"pid\": proc.pid,\n \"return_code\": proc.returncode,\n \"runner_pid\": os.getpid(),\n \"timestamp\": time.time(),\n }\n\n @contextlib.contextmanager\n def wrap(self, command, run_id, proc):\n with open(self.filename, \"w\") as fh:\n yaml.safe_dump(\n self.get_content(\n run_id=run_id,\n command=command,\n proc=proc,\n ),\n fh,\n explicit_start=True,\n width=1000000,\n )\n try:\n yield\n finally:\n with open(self.filename, \"a\") as fh:\n yaml.safe_dump(\n self.get_content(\n run_id=run_id,\n command=command,\n proc=proc,\n ),\n fh,\n explicit_start=True,\n width=1000000,\n )\n\n\ndef validate_output_dir(path):\n if os.path.isdir(path):\n if not os.access(path, os.W_OK):\n raise OSError(\"Output dir %s not writable\" % path)\n return\n else:\n try:\n os.makedirs(path)\n except OSError:\n raise OSError(\"Could not create output dir %s\" % path)\n\n\ndef build_environment(run_id, original_env=None):\n if original_env is None:\n original_env = dict(os.environ)\n try:\n namespace, job, run_num, action = run_id.split(\".\", maxsplit=3)\n except ValueError:\n # if we can't parse the run_id, we don't want to abort, so just\n # set these semi-arbitrarily\n namespace, job, run_num, action = [\"UNKNOWN\"] * 4\n\n new_env = dict(original_env)\n new_env[\"TRON_JOB_NAMESPACE\"] = namespace\n new_env[\"TRON_JOB_NAME\"] = job\n new_env[\"TRON_RUN_NUM\"] = run_num\n new_env[\"TRON_ACTION\"] = action\n\n logging.debug(new_env)\n return new_env\n\n\ndef build_labels(\n run_id: str,\n original_labels: dict[str, str] | None = None,\n attempt_number: int | None = None,\n) -> dict[str, str]:\n if original_labels is None:\n original_labels = {}\n\n try:\n # reminder: the format here is \"namespace.job.run_num.action\"\n _, _, run_num, _ = run_id.split(\".\", maxsplit=3)\n except ValueError:\n # if we can't parse the run_id, we don't want to abort, so just\n # set these semi-arbitrarily\n run_num = \"UNKNOWN\"\n\n new_labels = dict(original_labels)\n new_labels[\"tron.yelp.com/run_num\"] = run_num\n if attempt_number is not None:\n new_labels[\"tron.yelp.com/attempt_number\"] = str(attempt_number)\n\n return new_labels\n\n\ndef run_proc(output_path, command, run_id, proc):\n logging.warning(f\"{run_id} running as pid {proc.pid}\")\n status_file = StatusFile(os.path.join(output_path, STATUS_FILE))\n with status_file.wrap(\n command=command,\n run_id=run_id,\n proc=proc,\n ):\n returncode = proc.wait()\n logging.warning(f\"pid {proc.pid} exited with returncode {returncode}\")\n return returncode\n\n\ndef parse_args():\n parser = argparse.ArgumentParser(description=\"Action Runner for Tron\")\n parser.add_argument(\n \"output_dir\",\n help=\"The directory to store the state of the action run\",\n )\n parser.add_argument(\n \"command\",\n help=\"the command to run\",\n )\n parser.add_argument(\n \"run_id\",\n help=\"run_id of the action\",\n )\n return parser.parse_args()\n\n\ndef run_command(command, run_id):\n return subprocess.Popen(\n command,\n shell=True,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n env=build_environment(run_id=run_id),\n )\n\n\ndef stream(source, dst):\n is_connected = True\n logging.warning(f\"streaming {source.name} to {dst.name}\")\n for line in iter(source.readline, b\"\"):\n if is_connected:\n try:\n dst.write(line.decode(\"utf-8\"))\n dst.flush()\n logging.warning(f\"{dst.name}: {line}\")\n except Exception as e:\n logging.warning(f\"failed writing to {dst}: {e}\")\n logging.warning(f\"{dst.name}: {line}\")\n is_connected = False\n else:\n logging.warning(f\"{dst.name}: {line}\")\n is_connected = False\n\n\ndef configure_logging(run_id, output_dir):\n output_file = os.path.join(output_dir, f\"{run_id}-{os.getpid()}.log\")\n logging.basicConfig(\n filename=output_file,\n format=\"%(asctime)s %(levelname)s %(message)s\",\n datefmt=\"%Y-%m-%dT%H:%M:%S%z\",\n )\n\n\ndef main():\n args = parse_args()\n validate_output_dir(args.output_dir)\n configure_logging(run_id=args.run_id, output_dir=args.output_dir)\n proc = run_command(command=args.command, run_id=args.run_id)\n threads = [\n threading.Thread(target=stream, args=p, daemon=True)\n for p in [(proc.stdout, sys.stdout), (proc.stderr, sys.stderr)]\n ]\n for t in threads:\n t.start()\n returncode = run_proc(\n output_path=args.output_dir,\n run_id=args.run_id,\n command=args.command,\n proc=proc,\n )\n\n for t in threads:\n t.join()\n\n return returncode\n\n\nif __name__ == \"__main__\":\n sys.exit(main())\n" }, { "path": "tron/bin/action_status.py", "content": "#!/usr/bin/env python3.10\nimport argparse\nimport logging\nimport os\nimport signal\n\nfrom tron import yaml\n\nlog = logging.getLogger(\"tron.action_status\")\n\nSTATUS_FILE = \"status\"\n\n\ndef get_field(field, status_file):\n docs = yaml.load_all(status_file.read())\n content = list(docs)[-1]\n return content.get(field)\n\n\ndef print_status_file(status_file):\n for line in status_file.readlines():\n print(yaml.load(line))\n\n\ndef send_signal(signal_num, status_file):\n pid = get_field(\"pid\", status_file)\n if pid:\n try:\n os.killpg(os.getpgid(pid), signal_num)\n except OSError as e:\n msg = \"Failed to signal %s with %s: %s\"\n raise SystemExit(msg % (pid, signal_num, e))\n\n\ncommands = {\n \"print\": print_status_file,\n \"pid\": lambda statusfile: print(get_field(\"pid\", statusfile)),\n \"return_code\": lambda statusfile: print(get_field(\"return_code\", statusfile)),\n \"terminate\": lambda statusfile: send_signal(signal.SIGTERM, statusfile),\n \"kill\": lambda statusfile: send_signal(signal.SIGKILL, statusfile),\n}\n\n\ndef parse_args():\n parser = argparse.ArgumentParser(description=\"Action Status for Tron\")\n parser.add_argument(\n \"output_dir\",\n help=\"The directory where the state of the action run is\",\n )\n parser.add_argument(\n \"command\",\n help=\"the command to run\",\n )\n parser.add_argument(\n \"run_id\",\n help=\"run_id of the action\",\n )\n return parser.parse_args()\n\n\ndef run_command(command, status_file):\n commands[command](status_file)\n\n\ndef main():\n logging.basicConfig()\n args = parse_args()\n with open(os.path.join(args.output_dir, STATUS_FILE)) as f:\n run_command(args.command, f)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "tron/bin/check_tron_datastore_staleness.py", "content": "#!/usr/bin/env python3.10\nimport argparse\nimport logging\nimport os\nimport sys\nimport time\n\nimport pytz\n\nfrom tron.config import manager\nfrom tron.config import schema\nfrom tron.serialize.runstate.statemanager import PersistenceManagerFactory\n\n# Default values for arguments\nDEFAULT_WORKING_DIR = \"/var/lib/tron/\"\nDEFAULT_CONF_PATH = \"config/\"\nDEFAULT_STALENESS_THRESHOLD = 1800\nlog = logging.getLogger(\"check_tron_datastore_staleness\")\n\n\ndef get_last_run_time(job):\n \"\"\"\n Get all sorted timestamps, and only count the actions that actually ran\n \"\"\"\n timestamps = []\n job_runs = job[\"runs\"]\n for run in job_runs:\n for action in run[\"runs\"]:\n if action.get(\"start_time\") and action.get(\"state\") != \"scheduled\":\n timestamps.append(action.get(\"start_time\"))\n return max(timestamps) if timestamps else None\n\n\ndef parse_cli():\n parser = argparse.ArgumentParser()\n parser.add_argument(\n \"-w\",\n \"--working-dir\",\n default=DEFAULT_WORKING_DIR,\n help=\"Working directory for the Tron daemon, default %(default)s\",\n )\n parser.add_argument(\n \"-c\",\n \"--config-path\",\n default=DEFAULT_CONF_PATH,\n help=\"File path to the Tron configuration file\",\n )\n parser.add_argument(\n \"--job-name\",\n required=True,\n help=\"The job name to read timestamp from\",\n )\n parser.add_argument(\n \"--staleness-threshold\",\n default=DEFAULT_STALENESS_THRESHOLD,\n help=\"how long (in seconds) to wait to alert after the last timestamp of the job\",\n )\n args = parser.parse_args()\n args.working_dir = os.path.abspath(args.working_dir)\n args.config_path = os.path.join(\n args.working_dir,\n args.config_path,\n )\n return args\n\n\ndef read_config(args):\n return manager.ConfigManager(args.config_path).load().get_master().state_persistence\n\n\ndef main():\n # Fetch configs. You can find the arguments in puppet.\n args = parse_cli()\n persistence_config = read_config(args)\n store_type = schema.StatePersistenceTypes(persistence_config.store_type)\n job_name = args.job_name\n\n # Alert for DynamoDB\n if store_type == schema.StatePersistenceTypes.dynamodb:\n # Fetch job state from dynamodb\n state_manager = PersistenceManagerFactory.from_config(persistence_config)\n try:\n job = state_manager.restore(job_names=[job_name])[\"job_state\"][job_name]\n except Exception as e:\n logging.exception(f\"UNKN: Failed to retreive status for job {job_name} due to {e}\")\n sys.exit(3)\n\n # Exit if the job never runs.\n last_run_time = get_last_run_time(job)\n if not last_run_time:\n logging.error(\n f\"WARN: No last run for {job_name} found. If the job was just added, it might take some time for it to run\"\n )\n sys.exit(1)\n\n # Alert if timestamp is not updated after staleness_threshold\n stateless_for_secs = time.time() - last_run_time.astimezone(pytz.utc).timestamp()\n if stateless_for_secs > args.staleness_threshold:\n logging.error(f\"CRIT: {job_name} has not been updated in DynamoDB for {stateless_for_secs} seconds\")\n sys.exit(2)\n else:\n logging.info(f\"OK: DynamoDB is up to date. It's last updated at {last_run_time}\")\n sys.exit(0)\n # Alert for BerkeleyDB\n elif store_type == schema.StatePersistenceTypes.shelve:\n os.execl(\n \"/usr/lib/nagios/plugins/check_file_age\",\n \"/nail/tron/tron_state\",\n \"-w\",\n str(args.staleness_threshold),\n \"-c\",\n str(args.staleness_threshold),\n )\n else:\n logging.exception(f\"UNKN: Not designed to check this type of datastore: {store_type}\")\n sys.exit(3)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "tron/bin/check_tron_jobs.py", "content": "#!/usr/bin/env python3.10\nimport datetime\nimport logging\nimport pprint\nimport sys\nimport time\nfrom collections import defaultdict\nfrom enum import Enum\n\nimport pytimeparse # type: ignore[import-untyped] # no stubs or py.typed marker; likely want to move off of this\nfrom pyrsistent import m\nfrom pyrsistent import pmap\nfrom pysensu_yelp import send_event\n\nfrom tron.commands import cmd_utils\nfrom tron.commands import display\nfrom tron.commands.client import Client\nfrom tron.commands.client import get_object_type_from_identifier\nfrom tron.utils.logreader import get_superregion\n\nPRECIOUS_JOB_ATTR = \"check_that_every_day_has_a_successful_run\"\nNUM_PRECIOUS = 7\n\nlog = logging.getLogger(\"check_tron_jobs\")\n\n_run_interval = None\n\n\nclass State(Enum):\n SUCCEEDED = \"succeeded\"\n FAILED = \"failed\"\n STUCK = \"stuck\"\n NO_RUN_YET = \"no_run_yet\"\n NO_RUNS_TO_CHECK = \"no_runs_to_check\"\n UNKNOWN = \"unknown\"\n SKIPPED = \"skipped\"\n\n\ndef parse_cli():\n parser = cmd_utils.build_option_parser()\n parser.add_argument(\n \"--dry-run\",\n action=\"store_true\",\n default=False,\n help=\"Don't actually send alerts out. Defaults to %(default)s\",\n )\n parser.add_argument(\n \"--job\",\n default=None,\n help=\"Check a particular job. If unset checks all jobs\",\n )\n parser.add_argument(\n \"--run-interval\",\n help=\"Run interval for this monitoring script. This is used to \"\n \"calculate realert and alert_after setting. \"\n \"Default to %(default)s (seconds)\",\n type=int,\n dest=\"run_interval\",\n default=300,\n )\n parser.add_argument(\n \"--skip-sensu-failure-logging\",\n help=\"Skip including stdout/stderr logs in alerts for failed jobs\",\n action=\"store_true\",\n dest=\"skip_sensu_failure_logging\",\n default=False,\n )\n args = parser.parse_args()\n return args\n\n\ndef _timestamp_to_timeobj(timestamp):\n return time.strptime(timestamp, \"%Y-%m-%d %H:%M:%S\")\n\n\ndef _timestamp_to_shortdate(timestamp, separator=\".\"):\n return time.strftime(\n \"%Y{0}%m{0}%d\".format(separator),\n _timestamp_to_timeobj(timestamp),\n )\n\n\ndef compute_check_result_for_job_runs(client, job, job_content, url_index, hide_stderr=False):\n cluster = client.cluster_name\n kwargs = {}\n if job_content is None:\n kwargs[\"output\"] = f\"OK: {job['name']} was just added and hasn't run yet on {cluster}.\"\n kwargs[\"status\"] = 0\n return kwargs\n\n relevant_job_run, last_state = get_relevant_run_and_state(job_content)\n if relevant_job_run is None:\n kwargs[\"output\"] = (\n f\"CRIT: {job['name']} hasn't had a successful \" f\"run yet on {cluster}.\\n{pretty_print_job(job_content)}\"\n )\n kwargs[\"status\"] = 2\n return kwargs\n else: # if no run scheduled, no run_time available\n relevant_job_run_date = _timestamp_to_shortdate(\n relevant_job_run[\"run_time\"],\n )\n\n # A job_run is like MASTER.foo.1\n job_run_id = relevant_job_run[\"id\"]\n\n # A job action is like MASTER.foo.1.step1\n actions_expected_runtime = job_content.get(\"actions_expected_runtime\", {})\n relevant_action = get_relevant_action(\n action_runs=relevant_job_run[\"runs\"],\n last_state=last_state,\n actions_expected_runtime=actions_expected_runtime,\n )\n action_run_id = get_object_type_from_identifier(\n url_index,\n relevant_action[\"id\"],\n )\n\n if last_state in (State.STUCK, State.FAILED, State.UNKNOWN):\n if _skip_sensu_failure_logging:\n job_run_url = \"/\".join(job_run_id.rsplit(\".\", 1))\n tronweb_url = f\"http://y/tron-{get_superregion()}/#job/{job_run_url}\"\n stderr_default = f\"Please visit {tronweb_url} for stderr details.\"\n action_run_details = {}\n else:\n stderr_default = \"(No stderr available)\"\n action_run_details = client.action_runs(action_run_id.url, num_lines=10)\n else:\n action_run_details = {}\n\n if last_state == State.SUCCEEDED:\n prefix = f\"OK: The last job ({job_run_id}) run succeeded on {cluster}. Will watch future or in progress runs for the next failure\"\n status = 0\n stderr = \"\"\n elif last_state == State.NO_RUNS_TO_CHECK:\n prefix = f\"OK: The job {job['name']} is new and/or has no runs to check on {cluster}\"\n status = 0\n stderr = \"\"\n elif last_state == State.SKIPPED:\n prefix = f\"OK: The last job ({job_run_id}) run was skipped on {cluster}. Will watch future or in progress runs for the next failure\"\n status = 0\n stderr = \"\"\n elif last_state == State.STUCK:\n if job[\"monitoring\"].get(\"page_for_expected_runtime\", False):\n level = \"CRIT\"\n status = 2\n else:\n level = \"WARN\"\n status = 1\n prefix = f\"{level}: Job {job_run_id} exceeded expected runtime or still running when next job is scheduled on {cluster}\"\n stderr = \"\\n\".join(action_run_details.get(\"stderr\", [stderr_default]))\n elif last_state == State.FAILED:\n prefix = f\"CRIT: The last job run ({job_run_id}) failed on {cluster}!\"\n status = 2\n stderr = \"\\n\".join(action_run_details.get(\"stderr\", [stderr_default]))\n elif last_state == State.UNKNOWN:\n prefix = f\"CRIT: Job {job_run_id} has gone 'unknown' and might need manual intervention on {cluster}\"\n status = 2\n stderr = \"\"\n else:\n prefix = f\"UNKNOWN: Job {job_run_id} is in a state that check_tron_jobs doesn't understand\"\n status = 3\n stderr = \"\"\n\n if hide_stderr:\n stderr = \"\"\n\n precious_runs_note = \"\"\n if job[\"monitoring\"].get(PRECIOUS_JOB_ATTR, False) and status != 0:\n precious_runs_note = f\"Note: This alert is the run for {relevant_job_run_date}. A resolve event will not occur until a job run for this date succeeds.\\n\"\n\n kwargs[\"output\"] = (\n f\"{prefix}\\n\"\n f\"{stderr}\\n\"\n f\"The latest run, {relevant_job_run['id']} {relevant_job_run['state']}\\n\"\n f\"{precious_runs_note}\"\n )\n if action_run_details:\n kwargs[\"output\"] += \"\\nHere is the last action:\\n\" f\"{pretty_print_actions(action_run_details)}\\n\\n\"\n kwargs[\"output\"] += (\n \"And the job run view:\\n\"\n f\"{pretty_print_job_run(relevant_job_run)}\\n\\n\"\n \"Here is the whole job view for context:\\n\"\n f\"{pretty_print_job(job_content)}\"\n )\n kwargs[\"status\"] = status\n return kwargs\n\n\ndef pretty_print_job(job_content):\n return display.format_job_details(job_content)\n\n\ndef pretty_print_job_run(job_run):\n display_action = display.DisplayActionRuns()\n return display_action.format(job_run)\n\n\ndef pretty_print_actions(action_run):\n return display.format_action_run_details(action_run)\n\n\ndef get_relevant_run_and_state(job_content):\n # The order of job run to check is as follows:\n # 1. The scheduled but hasn't run one checked first\n # 2. Then currently running ones are always checked (in case an action is failed/unknown)\n # 3. If there are multiple running ones, then most recent run_time wins\n # 4. If nothing is currently running, then most recent end_time wins\n job_runs = sorted(\n job_content.get(\"runs\", []),\n key=lambda k: (k[\"end_time\"] is None, k[\"end_time\"], k[\"run_time\"]),\n reverse=True,\n )\n if len(job_runs) == 0:\n return None, State.NO_RUN_YET\n\n job_expected_runtime = job_content.get(\"expected_runtime\", None)\n actions_expected_runtime = job_content.get(\"actions_expected_runtime\", {})\n stuck_run = is_job_stuck(\n job_runs=job_runs,\n job_expected_runtime=job_expected_runtime,\n actions_expected_runtime=actions_expected_runtime,\n allow_overlap=job_content.get(\"allow_overlap\", False),\n queueing=job_content.get(\"queueing\", True),\n )\n for run in job_runs:\n state = run.get(\"state\", \"unknown\")\n if state in [\"failed\", \"succeeded\", \"unknown\", \"skipped\"]:\n return run, State(state)\n elif state in [\"running\", \"waiting\", \"starting\"]:\n action_state = is_action_failed_or_unknown(run)\n if action_state != State.SUCCEEDED:\n return run, action_state\n elif stuck_run is not None:\n return stuck_run, State.STUCK\n return job_runs[0], State.NO_RUNS_TO_CHECK\n\n\ndef is_action_failed_or_unknown(job_run):\n for run in job_run.get(\"runs\", []):\n if run.get(\"state\", None) in [\"failed\", \"unknown\"]:\n return State(run.get(\"state\"))\n return State.SUCCEEDED\n\n\ndef is_job_stuck(\n job_runs,\n job_expected_runtime,\n actions_expected_runtime,\n allow_overlap,\n queueing,\n):\n next_run_time = None\n for job_run in job_runs:\n states_to_check = {\"running\", \"waiting\", \"starting\"}\n if job_run.get(\"state\", \"unknown\") in states_to_check:\n if is_job_run_exceeding_expected_runtime(\n job_run,\n job_expected_runtime,\n ):\n return job_run\n # check if it is still running at next scheduled job run time\n if not allow_overlap and queueing and next_run_time:\n difftime = _timestamp_to_timeobj(next_run_time)\n if time.time() > time.mktime(difftime):\n return job_run\n for action_run in job_run.get(\"runs\", []):\n if is_action_run_exceeding_expected_runtime(\n action_run,\n actions_expected_runtime,\n ):\n return job_run\n\n next_run_time = job_run.get(\"run_time\", None)\n return None\n\n\ndef is_job_run_exceeding_expected_runtime(job_run, job_expected_runtime):\n states_to_check = {\"running\", \"waiting\", \"starting\"}\n if (\n job_expected_runtime is not None\n and job_run.get(\n \"state\",\n \"unknown\",\n )\n in states_to_check\n ):\n duration_seconds = pytimeparse.parse(job_run.get(\"duration\", \"\"))\n if duration_seconds and duration_seconds > job_expected_runtime:\n return True\n return False\n\n\ndef is_action_run_exceeding_expected_runtime(\n action_run,\n actions_expected_runtime,\n):\n states_to_check = [\"running\", \"starting\"]\n if action_run.get(\"state\", \"unknown\") in states_to_check:\n action_name = action_run.get(\"action_name\", None)\n if action_name in actions_expected_runtime and actions_expected_runtime[action_name] is not None:\n duration_seconds = pytimeparse.parse(\n action_run.get(\"duration\", \"\"),\n )\n if duration_seconds > actions_expected_runtime[action_name]:\n return True\n return False\n\n\ndef get_relevant_action(*, action_runs, last_state, actions_expected_runtime):\n stuck_action_run_candidate = None\n for action_run in reversed(action_runs):\n action_state = action_run.get(\"state\", \"unknown\")\n try:\n if State(action_state) == last_state:\n return action_run\n except ValueError:\n if last_state == State.STUCK:\n if is_action_run_exceeding_expected_runtime(\n action_run,\n actions_expected_runtime,\n ):\n return action_run\n if action_state in {\"running\", \"starting\"}:\n stuck_action_run_candidate = action_run\n return stuck_action_run_candidate or action_runs[-1]\n\n\ndef guess_realert_every(job):\n try:\n job_next_run = job.get(\"next_run\", None)\n if job_next_run is None:\n return -1\n job_runs = job.get(\"runs\", [])\n job_runs_started = [\n run.get(\"start_time\") or run.get(\"run_time\")\n for run in job_runs\n if run.get(\"start_time\") or run.get(\"run_time\") and run.get(\"run_time\") != job_next_run\n ]\n if len(job_runs_started) == 0:\n return -1\n job_previous_run = max(\n job_runs_started,\n )\n time_diff = time.mktime(_timestamp_to_timeobj(job_next_run)) - time.mktime(\n _timestamp_to_timeobj(job_previous_run)\n )\n realert_every = max(int(time_diff / _run_interval), 1)\n except Exception as e:\n log.warning(f\"guess_realert_every failed: {e}\")\n return -1\n return realert_every\n\n\ndef get_earliest_run_time_to_check(job_content, interval):\n if not job_content[\"runs\"]:\n return None\n\n earliest_run_time = min(time.mktime(_timestamp_to_timeobj(run[\"run_time\"])) for run in job_content[\"runs\"])\n return max(\n earliest_run_time,\n time.time() - datetime.timedelta(**{f\"{interval}s\": NUM_PRECIOUS - 1}).total_seconds(),\n )\n\n\ndef sort_runs_by_interval(job_content, interval=\"day\", until=None):\n \"\"\"Sorts a job's runs by a time interval (day, hour, minute, or second),\n according to a job run's run time.\n \"\"\"\n interval_formats = {\n \"day\": \"%Y.%m.%d\",\n \"hour\": \"%Y.%m.%d-%H\",\n \"minute\": \"%Y.%m.%d-%H.%M\",\n \"second\": \"%Y.%m.%d-%H.%M.%S\",\n }\n fmt = interval_formats[interval]\n\n run_buckets = defaultdict(list)\n if job_content is not None:\n if not until:\n until = time.time() # can't set in default arg\n earliest_run_time = get_earliest_run_time_to_check(job_content, interval) or until\n\n # We add all dates by interval between our earliest run_time and now,\n # allowing functions downstream to see if some dates had no runs\n start = datetime.datetime.fromtimestamp(earliest_run_time)\n end = datetime.datetime.fromtimestamp(until)\n step = datetime.timedelta(**{f\"{interval}s\": 1})\n\n # We compare the strings _after_ we've converted to the final format to make\n # sure we don't miss something due to off-by-one/weird DST bugs, etc\n while start.strftime(fmt) <= end.strftime(fmt):\n run_buckets[start.strftime(fmt)] = []\n start += step\n\n # Bucket runs by interval\n for run in job_content[\"runs\"]:\n run_time = time.strftime(\n interval_formats[interval],\n _timestamp_to_timeobj(run[\"run_time\"]),\n )\n if run_time not in run_buckets:\n continue\n run_buckets[run_time].append(run)\n return dict(run_buckets)\n\n\ndef compute_check_result_for_job(client, job, url_index):\n kwargs = m(\n name=f\"check_tron_job.{job['name']}\",\n source=client.cluster_name,\n )\n if \"realert_every\" not in kwargs:\n kwargs = kwargs.set(\"realert_every\", guess_realert_every(job))\n kwargs = kwargs.set(\"check_every\", f\"{_run_interval}s\")\n\n # We want to prevent a monitoring config from setting the check_every\n # attribute, since one config should not dictate how often this script runs\n sensu_kwargs = (\n pmap(job[\"monitoring\"])\n .discard(PRECIOUS_JOB_ATTR)\n .discard(\"check_every\")\n .discard(\"page_for_expected_runtime\")\n .discard(\"check_oom_events\")\n )\n kwargs = kwargs.update(sensu_kwargs)\n hide_stderr = kwargs.get(\"hide_stderr\", False)\n kwargs_list = []\n if job[\"status\"] == \"disabled\":\n kwargs = kwargs.set(\n \"output\",\n f\"OK: {job['name']} is disabled and won't be checked.\",\n )\n kwargs = kwargs.set(\"status\", 0)\n kwargs_list.append(kwargs)\n else:\n # The job is not disabled, therefore we have to look at its run history\n tron_id = get_object_type_from_identifier(url_index, job[\"name\"])\n job_content = pmap(\n client.job(\n tron_id.url,\n include_action_runs=True,\n ),\n )\n\n if job[\"monitoring\"].get(PRECIOUS_JOB_ATTR, False):\n dated_runs = sort_runs_by_interval(job_content, interval=\"day\")\n else:\n dated_runs = {\"\": job_content[\"runs\"]}\n\n for date, runs in dated_runs.items():\n results = compute_check_result_for_job_runs(\n job=job,\n job_content=job_content.set(\"runs\", runs),\n client=client,\n url_index=url_index,\n hide_stderr=hide_stderr,\n )\n dated_kwargs = kwargs.update(results)\n if date: # if empty date, leave job name alone\n dated_kwargs = dated_kwargs.set(\n \"name\",\n f\"{kwargs['name']}-{date}\",\n )\n kwargs_list.append(dated_kwargs)\n\n return [dict(kws) for kws in kwargs_list]\n\n\ndef check_job(job, client, url_index):\n if job.get(\"monitoring\", {}) == {}:\n log.debug(f\"Not checking {job['name']}, no monitoring metadata setup.\")\n return\n if job.get(\"monitoring\").get(\"team\", None) is None:\n log.debug(f\"Not checking {job['name']}, no team specified\")\n return\n log.info(f\"Checking {job['name']}\")\n return compute_check_result_for_job(job=job, client=client, url_index=url_index)\n\n\ndef check_job_result(job, client, url_index, dry_run):\n results = check_job(job, client, url_index)\n if not results:\n return\n\n for result in results:\n if dry_run:\n log.info(\"Would have sent this event to sensu: \")\n log.info(pprint.pformat(result))\n else:\n log.debug(f\"Sending event: {pprint.pformat(result)}\")\n if \"runbook\" not in result:\n result[\"runbook\"] = (\n \"No runbook specified. Please specify a runbook in the monitoring section of the job definition.\",\n )\n send_event(**result)\n\n\ndef main():\n args = parse_cli()\n cmd_utils.setup_logging(args)\n cmd_utils.load_config(args)\n client = Client(args.server, args.cluster_name)\n\n error_code = 0\n global _run_interval\n _run_interval = args.run_interval\n\n global _skip_sensu_failure_logging\n _skip_sensu_failure_logging = args.skip_sensu_failure_logging\n\n url_index = client.index()\n if args.job is None:\n jobs = client.jobs(include_job_runs=True)\n for job in jobs:\n try:\n check_job_result(job=job, client=client, url_index=url_index, dry_run=args.dry_run)\n except Exception as e:\n log.warning(f\"check job result fails for job {job.get('name', '')}: {e}\")\n error_code = 1\n else:\n job_url = client.get_url(args.job)\n job = client.job_runs(job_url)\n check_job_result(job=job, client=client, url_index=url_index, dry_run=args.dry_run)\n\n return error_code\n\n\nif __name__ == \"__main__\":\n sys.exit(main())\n" }, { "path": "tron/bin/get_tron_metrics.py", "content": "#!/usr/bin/env python3.10\n#\n# get_tron_metrics.py\n# This script is designed to retrieve metrics from Tron via its API and send\n# send them to meteorite.\nimport logging\nimport pprint\nimport subprocess\nimport sys\nimport textwrap\n\nfrom tron.commands import cmd_utils\nfrom tron.commands.client import Client\n\nlog = logging.getLogger(\"get_tron_metrics\")\n\n\ndef parse_cli():\n parser = cmd_utils.build_option_parser()\n parser.description = \"Collects metrics from Tron via its API and forwards them to \" \"meteorite.\"\n parser.add_argument(\n \"--dry-run\",\n action=\"store_true\",\n default=False,\n help=\"Don't actually send metrics out. Defaults: %(default)s\",\n )\n args = parser.parse_args()\n return args\n\n\ndef check_bin_exists(bin):\n \"\"\"\n Checks if an executable binary exists\n\n :param bin: (str) Name of the executable; could be a path to one\n \"\"\"\n return (\n subprocess.call(\n [\"which\", bin],\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n )\n == 0\n )\n\n\ndef send_data_metric(name, metric_type, value, dimensions={}, dry_run=False):\n \"\"\"\n Sends a single data point to meteorite via bash command\n\n :param name: (str) Name of the metric\n :param metric_type: (str) Type of the meteorite metric. Must be in\n METEORITE_TYPES\n :param value: (float) Value of the metric\n :param dimensions: (dict) Metric dimensions as key-value pairs\n :param dry_run: (bool) Whether or not to send metrics to meteorite\n \"\"\"\n if dry_run:\n metric_args = dict(\n name=name,\n metric_type=metric_type,\n value=value,\n dimensions=dimensions,\n )\n log.info(\n f\"Would have sent this to meteorite:\\n\" f\"{pprint.pformat(metric_args)}\",\n )\n return\n\n cmd = [\"meteorite\", \"data\", \"-v\", name, metric_type, str(value)]\n for k, v in dimensions.items():\n cmd.extend([\"-d\", f\"{k}:{v}\"])\n\n process = subprocess.Popen(\n cmd,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n )\n output, error = process.communicate()\n output = output.decode(\"utf-8\").rstrip()\n error = error.decode(\"utf-8\").rstrip()\n\n if process.returncode != 0:\n log.error(\n \"Meteorite failed with:\\n\" f\"{textwrap.indent(error, ' ')}\",\n )\n else:\n log.debug(f\"From meteorite: {output}\")\n\n\ndef send_counter(name, **kwargs):\n send_data_metric(\n name=name,\n metric_type=\"counter\",\n value=kwargs.pop(\"count\"),\n dimensions=kwargs.pop(\"dimensions\", {}),\n dry_run=kwargs.pop(\"dry_run\", False),\n )\n\n\ndef send_gauge(name, **kwargs):\n send_data_metric(\n name=name,\n metric_type=\"gauge\",\n value=kwargs.pop(\"value\"),\n dimensions=kwargs.pop(\"dimensions\", {}),\n dry_run=kwargs.pop(\"dry_run\", False),\n )\n\n\ndef send_meter(name, **kwargs):\n send_counter(name, **kwargs) # We ignore mX_rate args\n\n\ndef send_histogram(name, **kwargs):\n for k in [\"p50\", \"p75\", \"p95\", \"p99\"]: # Only send p50-99\n gauge_name = f\"{name}.{k}\"\n kwargs[\"value\"] = kwargs[k] # set for gauge\n send_gauge(gauge_name, **kwargs)\n\n\ndef send_timer(name, **kwargs):\n # We mirror the metrics implementation in Tron by splitting timer into a\n # meter and a histogram\n send_meter(name, **kwargs)\n send_histogram(name, **kwargs)\n\n\n_METRIC_SENDERS = {\n \"counter\": send_counter,\n \"gauge\": send_gauge,\n \"meter\": send_meter,\n \"histogram\": send_histogram,\n \"timer\": send_timer,\n}\n\n\ndef send_metrics(metrics, cluster=None, dry_run=False):\n \"\"\"\n Send metrics via meteorite\n\n :param metrics: Dictionary of metrics types and their data\n \"\"\"\n for metric_type, data in metrics.items():\n for kwargs in data:\n name = kwargs.pop(\"name\")\n kwargs[\"dry_run\"] = dry_run\n\n if cluster:\n dimensions = kwargs.get(\"dimensions\", {})\n dimensions[\"tron_cluster\"] = cluster\n kwargs[\"dimensions\"] = dimensions\n\n _METRIC_SENDERS[metric_type](name, **kwargs)\n\n\ndef main():\n args = parse_cli()\n cmd_utils.setup_logging(args)\n cmd_utils.load_config(args)\n client = Client(args.server, args.cluster_name)\n\n if check_bin_exists(\"meteorite\"):\n metrics = client.metrics()\n send_metrics(metrics, cluster=client.cluster_name, dry_run=args.dry_run)\n else:\n log.error(\"'meteorite' was not found\")\n\n\nif __name__ == \"__main__\":\n sys.exit(main())\n" }, { "path": "tron/bin/recover_batch.py", "content": "#!/usr/bin/env python3.10\nimport argparse\nimport logging\nimport signal\nimport sys\nfrom queue import Queue\n\nimport psutil\nfrom twisted.internet import inotify\nfrom twisted.internet import reactor\nfrom twisted.python import filepath\n\nfrom tron import yaml\n\nlog = logging.getLogger(\"tron.recover_batch\")\n\n\nclass StatusFileWatcher:\n \"\"\"\n Watches the status file produced by action runners\n \"\"\"\n\n def __init__(self, to_watch, callback):\n notifier = inotify.INotify()\n notifier.startReading()\n notifier.watch(filepath.FilePath(to_watch), callbacks=[callback])\n\n\ndef parse_args():\n parser = argparse.ArgumentParser(\n description=\"Check if a action runner has exited; wait otherwise\",\n )\n parser.add_argument(\"filepath\")\n return parser.parse_args()\n\n\ndef read_last_yaml_entries(filename):\n with open(filename) as f:\n lines = list(yaml.load_all(f))\n if not lines:\n entries = {}\n else:\n entries = lines[-1]\n return entries\n\n\ndef notify(notify_queue, ignored, filepath, mask):\n exit_code, error_message = get_exit_code(filepath.path)\n if exit_code is not None:\n reactor.stop()\n notify_queue.put((exit_code, error_message))\n\n\ndef get_exit_code(filepath):\n entries = read_last_yaml_entries(filepath)\n pid = entries.get(\"runner_pid\")\n return_code = entries.get(\"return_code\")\n exit_code, error_message = None, None\n\n if return_code is not None:\n if return_code < 0:\n # from the subprocess docs on the return code of a process:\n # \"A negative value -N indicates that the child was terminated by signal N (POSIX only).\"\n # We should always exit with a positive code, so we take the absolute value of the return code\n exit_code = abs(return_code)\n error_message = f\"Action run killed by signal {signal.Signals(exit_code).name}\"\n else:\n exit_code = return_code\n elif pid is None:\n log.warning(f\"Status file {filepath} didn't have a PID. Will watch the file for updates.\")\n elif not psutil.pid_exists(pid):\n exit_code = 1\n error_message = f\"Action runner pid {pid} no longer running. Assuming an exit of 1.\"\n\n return exit_code, error_message\n\n\ndef run(fpath):\n # Check if the process has already completed.\n # If it has, we don't expect any more updates.\n return_code, error_message = get_exit_code(fpath)\n if return_code is not None:\n if error_message is not None:\n log.warning(error_message)\n sys.exit(return_code)\n\n # If not, wait for updates to the file.\n notify_queue = Queue()\n StatusFileWatcher(\n fpath,\n lambda *args, **kwargs: notify(notify_queue, *args, **kwargs),\n )\n reactor.run()\n exit_code, error_message = notify_queue.get()\n if error_message is not None:\n log.warning(error_message)\n sys.exit(exit_code)\n\n\nif __name__ == \"__main__\":\n args = parse_args()\n run(args.filepath)\n" }, { "path": "tron/command_context.py", "content": "\"\"\"Command Context is how we construct the command line for a command which may\nhave variables that need to be rendered.\n\"\"\"\nimport operator\nimport re\nfrom functools import reduce\n\nfrom tron.utils import timeutils\n\n\ndef build_context(object, parent):\n \"\"\"Construct a CommandContext for object. object must have a property\n 'context_class'.\n \"\"\"\n return CommandContext(object.context_class(object), parent)\n\n\ndef build_filled_context(*context_objects):\n \"\"\"Create a CommandContext chain from context_objects, using a Filler\n object to pass to each CommandContext. Can be used to validate a format\n string.\n \"\"\"\n if not context_objects:\n return CommandContext()\n\n filler = Filler()\n\n def build(current, next):\n return CommandContext(next(filler), current)\n\n return reduce(build, context_objects, None)\n\n\nclass CommandContext:\n \"\"\"A CommandContext object is a wrapper around any object which has values\n to be used to render a command for execution. It looks up values by name.\n\n It's lookup order is:\n base[name],\n base.__getattr__(name),\n next[name],\n next.__getattr__(name)\n \"\"\"\n\n def __init__(self, base=None, next=None):\n \"\"\"\n base - Object to look for attributes in\n next - Next place to look for more pieces of context\n Generally this will be another instance of CommandContext\n \"\"\"\n self.base = base or {}\n self.next = next or {}\n\n def get(self, name, default=None):\n try:\n return self.__getitem__(name)\n except KeyError:\n return default\n\n def __getitem__(self, name):\n getters = [operator.itemgetter(name), operator.attrgetter(name)]\n for target in [self.base, self.next]:\n for getter in getters:\n try:\n return getter(target)\n except (KeyError, TypeError, AttributeError):\n pass\n\n raise KeyError(name)\n\n def __eq__(self, other):\n return self.base == other.base and self.next == other.next\n\n def __ne__(self, other):\n return not self == other\n\n\nclass JobContext:\n \"\"\"A class which exposes properties for rendering commands.\"\"\"\n\n def __init__(self, job):\n self.job = job\n\n @property\n def name(self):\n return self.job.name\n\n def __getitem__(self, item):\n date_name, date_spec = self._get_date_spec_parts(item)\n if not date_spec:\n raise KeyError(item)\n\n if date_name == \"last_success\":\n last_success = self.job.runs.last_success\n last_success = last_success.run_time if last_success else None\n\n time_value = timeutils.DateArithmetic.parse(\n date_spec,\n last_success,\n )\n if time_value:\n return time_value\n\n raise KeyError(item)\n\n def _get_date_spec_parts(self, name):\n parts = name.rsplit(\"#\", 1)\n if len(parts) != 2:\n return name, None\n return parts\n\n @property\n def namespace(self):\n return self.job.name.split(\".\")[0]\n\n\nclass JobRunContext:\n def __init__(self, job_run):\n self.job_run = job_run\n\n @property\n def runid(self):\n return self.job_run.id\n\n @property\n def manual(self):\n return str(self.job_run.manual).lower()\n\n @property\n def cleanup_job_status(self):\n \"\"\"Provide 'SUCCESS' or 'FAILURE' to a cleanup action context based on\n the status of the other steps\n \"\"\"\n if self.job_run.action_runs.is_failed:\n return \"FAILURE\"\n elif self.job_run.action_runs.is_complete_without_cleanup:\n return \"SUCCESS\"\n return \"UNKNOWN\"\n\n def __getitem__(self, name):\n \"\"\"\n This function attempts to parse any command context variable expressions\n that use shortdate or runid in the following order:\n 1) Attempt to parse date arithmetic syntax and apply to run_time unconditionally\n and, if unsuccessful falls to the next case\n 2) Attempts to parse a delta to apply to the current job runid - this is mostly\n meant to be used for jobs that rely on the output of the previous run, but\n this is not enforced in case someone can dream up another scenario where they\n want to do arbitrary deltas here.\n \"\"\"\n run_time = self.job_run.run_time\n time_value = timeutils.DateArithmetic.parse(name, run_time)\n if time_value:\n return time_value\n\n # this is a little weird, but enumerating the cases that should be parsed by timeutils is hard,\n # so we just unconditionally attempt to parse the name and then fallback to the runid special cases\n # rather than attempt to enumerate the timeutils cases\n elif name == \"runid\":\n # we could expand the logic below to handle this with the regex, but that\n # would make the code a little more complex for not much gain\n return self.runid\n elif \"runid\" in name:\n # we're really only expecting runid-1 for now but, as described in the docstring,\n # we're allowing arbitrary addition/subtration in case someone dreams up a use for\n # them\n match = re.match(r\"^runid([+-]\\d+)$\", name)\n if match:\n # self.runid here will be the job runid (e.g., NAMESPACE.SERVICE.RUN_NUMBER) - it will not\n # include an action name.\n # that said - all we need math-wise here is the run number, so we split on . and store the job name\n # so that we can re-consistitute the runid after doing math on the run number\n job_name, run_num = self.runid.rsplit(\".\", maxsplit=1)\n # NOTE: this will potentially return a runid for a job that will never exist - e.g., if you setup an\n # action that should only run after the previous jobrun's action has run for a job that has never run\n # before) - normally this will only be a problem for the very first run and users can easily tronctl start\n # the action to bootstrap things so we don't do any checking to see if the returned runid is valid\n return f\"{job_name}.{int(run_num) + int(match.groups()[0])}\"\n\n raise KeyError(name)\n\n\nclass ActionRunContext:\n \"\"\"Context object that gives us access to data about the action run.\"\"\"\n\n def __init__(self, action_run):\n self.action_run = action_run\n\n @property\n def actionname(self):\n return self.action_run.action_name\n\n @property\n def node(self):\n return self.action_run.node.hostname\n\n\nclass Filler:\n \"\"\"Filler object for using CommandContext during config parsing. This class\n is used as a substitute for objects that would be passed to Context objects.\n This allows the Context objects to be used directly for config validation.\n \"\"\"\n\n def __getattr__(self, _):\n return self\n\n def __str__(self):\n return \"%(...)s\"\n\n def __mod__(self, _):\n return self\n\n def __nonzero__(self):\n return False\n\n def __bool__(self):\n return False\n" }, { "path": "tron/commands/__init__.py", "content": "" }, { "path": "tron/commands/authentication.py", "content": "import os\nfrom typing import cast\n\nfrom tron.commands.cmd_utils import get_client_config\n\ntry:\n from vault_tools.oidc import get_instance_oidc_identity_token # type: ignore # library lacks py.typed marker\n from okta_auth import get_and_cache_jwt_default # type: ignore # library lacks py.typed marker\nexcept ImportError:\n\n def get_instance_oidc_identity_token(role: str, ecosystem: str | None = None) -> str:\n return \"\"\n\n def get_and_cache_jwt_default(client_id: str, refreshable: bool = False, force: bool = False) -> str:\n return \"\"\n\n\ndef get_sso_auth_token(no_cache: bool = False) -> str:\n \"\"\"Generate an authentication token for the calling user from the Single Sign On provider, if configured\"\"\"\n client_id = get_client_config().get(\"auth_sso_oidc_client_id\")\n return cast(str, get_and_cache_jwt_default(client_id, refreshable=True, force=no_cache)) if client_id else \"\"\n\n\ndef get_vault_auth_token() -> str:\n \"\"\"Generate an authentication token for the underlying instance via Vault\"\"\"\n vault_role = get_client_config().get(\"vault_api_auth_role\", \"service_authz\")\n return cast(str, get_instance_oidc_identity_token(vault_role))\n\n\ndef get_auth_token(no_cache: bool = False) -> str:\n \"\"\"Generate authentication token via Vault or Okta\"\"\"\n return get_vault_auth_token() if os.getenv(\"TRONCTL_VAULT_AUTH\") else get_sso_auth_token(no_cache)\n" }, { "path": "tron/commands/backfill.py", "content": "import asyncio\nimport datetime\nimport functools\nimport os\nimport pprint\nimport re\nimport signal\nimport sys\nfrom urllib.parse import urljoin\n\nfrom tron.commands import client\nfrom tron.commands import display\nfrom tron.commands.authentication import get_auth_token\nfrom tron.core.actionrun import ActionRun\n\nDEFAULT_MAX_PARALLEL_RUNS = 3\nLIMIT_MAX_PARALLEL_RUNS = 10\nDEFAULT_POLLING_INTERVAL_S = 10\n\n\ndef get_date_range(\n start_date: datetime.datetime,\n end_date: datetime.datetime,\n descending: bool = False,\n) -> list[datetime.datetime]:\n dates = []\n delta = end_date - start_date\n for days_to_add in range(delta.days + 1):\n dates.append(start_date + datetime.timedelta(days=days_to_add))\n if descending:\n dates.reverse()\n return dates\n\n\ndef print_backfill_cmds(job: str, date_strs: list[str]) -> None:\n print(f\"Please run the following {len(date_strs)} commands:\")\n print(\"\")\n for date in date_strs:\n print(f\"tronctl start {job} --run-date {date}\")\n print(\"\")\n print(\"Note that many jobs operate on the previous day's data.\")\n\n\ndef confirm_backfill(job: str, date_strs: list[str]) -> bool:\n print(\n f\"To backfill for the job '{job}', a job run will be created for each \"\n f\"of the following {len(date_strs)} dates:\"\n )\n pprint.pprint(date_strs)\n print(\"\")\n user_resp = input(\"Confirm? [y/n] \")\n\n if user_resp.lower() != \"y\":\n print(\"Aborted.\")\n return False\n else:\n print(\"\") # just for clean separation\n return True\n\n\nclass BackfillRun:\n NOT_STARTED_STATE = \"not started\"\n SUCCESS_STATES = {ActionRun.SUCCEEDED, ActionRun.CANCELLED, ActionRun.SKIPPED}\n MAX_SYNC_FAILURES = 5\n\n def __init__(self, tron_client: client.Client, job_id: client.TronObjectIdentifier, run_time: datetime.datetime):\n self.tron_client = tron_client\n self.job_id = job_id\n self.run_time = run_time\n self.run_name: str | None = None\n self.run_id: client.TronObjectIdentifier | None = None\n self.run_state = BackfillRun.NOT_STARTED_STATE\n\n @property\n def run_time_str(self) -> str:\n return self.run_time.date().isoformat()\n\n async def run_until_completion(self) -> str:\n \"\"\"Runs this job run until it finishes (i.e. reaches a terminal state).\"\"\"\n try:\n if await self.create():\n await self.sync_state()\n await self.watch_until_completion()\n except asyncio.CancelledError:\n await self.cancel()\n return self.run_state\n\n async def create(self) -> str | None:\n \"\"\"Creates job run for a specific date.\n\n Returns the name of the run, if it was created with no issues.\n \"\"\"\n # create the job run\n loop = asyncio.get_event_loop()\n response = await loop.run_in_executor(\n None,\n functools.partial(\n client.request,\n urljoin(self.tron_client.url_base, self.job_id.url),\n data=dict(command=\"start\", run_time=self.run_time),\n user_attribution=True,\n ),\n )\n\n # figure out its name\n if response.error:\n print(f\"Error: couldn't start job run for {self.run_time_str}: {response.content}\")\n else:\n # determine name of job run so that we can watch it\n # from tron.api.controller.JobController and tron.core.jobrun, the format\n # of the response result will be: \"Created JobRun:\"\n result = response.content.get(\"result\")\n match = re.match(r\"^Created JobRun:([-.\\w]+)$\", result)\n\n if match:\n self.run_name = match.groups(0)[0] # type: ignore[assignment] # mypy wrongly identifies self.run_name type as \"Union[str, int]\"\n self.run_state = ActionRun.STARTING\n print(f\"Job run '{self.run_name}' for {self.run_time_str} created\")\n else:\n print(\n f\"Warning: Job run for {self.run_time_str} created, but couldn't determine \"\n \"its name, so its state is considered to be unknown.\"\n )\n self.run_state = ActionRun.UNKNOWN\n\n return self.run_name\n\n async def get_run_id(self) -> client.TronObjectIdentifier | None:\n if not self.run_id:\n loop = asyncio.get_event_loop()\n try:\n self.run_id = await loop.run_in_executor(\n None,\n client.get_object_type_from_identifier,\n self.tron_client.index(),\n self.run_name,\n )\n\n except client.RequestError as e:\n print(f\"Error: couldn't get resource URL for job run '{self.run_name}': {e}\")\n\n return self.run_id\n\n async def sync_state(self) -> str:\n \"\"\"Syncs the local run state with that of the Tron server's.\n\n Returns the updated state.\n \"\"\"\n if not self.run_id:\n self.run_id = await self.get_run_id()\n\n if self.run_id:\n loop = asyncio.get_event_loop()\n try:\n # get the state of the run using the resource url\n resp_content = await loop.run_in_executor(\n None,\n functools.partial(\n self.tron_client.job_runs,\n urljoin(self.tron_client.url_base, self.run_id.url),\n include_runs=False,\n include_graph=False,\n ),\n )\n self.run_state = resp_content.get(\"state\", ActionRun.UNKNOWN)\n\n except (client.RequestError, AttributeError, ValueError) as e:\n print(f\"Error: couldn't get state for job run '{self.run_name}': {e}\")\n self.run_state = ActionRun.UNKNOWN\n\n return self.run_state\n\n async def watch_until_completion(self, poll_intv_s: int = DEFAULT_POLLING_INTERVAL_S) -> str:\n \"\"\"Watches this job run until it finishes.\n\n Returns the end state of the run.\n \"\"\"\n sync_failures = 0\n while self.run_state not in ActionRun.END_STATES:\n await asyncio.sleep(poll_intv_s)\n try:\n await self.sync_state()\n sync_failures = 0\n except Exception as e:\n print(f\"Issue syncing state for '{self.run_name}': {e}\", file=sys.stderr)\n sync_failures += 1\n if sync_failures > self.MAX_SYNC_FAILURES:\n raise\n\n print(f\"Job run '{self.run_name}' for {self.run_time_str} finished with state: {self.run_state}\")\n return self.run_state\n\n async def cancel(self) -> bool:\n \"\"\"Cancel this run if it is running.\n\n Returns whether or not the run was successfully cancelled\n \"\"\"\n if self.run_id:\n loop = asyncio.get_event_loop()\n response = await loop.run_in_executor(\n None,\n functools.partial(\n client.request,\n urljoin(self.tron_client.url_base, self.run_id.url),\n data=dict(command=\"cancel\"),\n user_attribution=True,\n ),\n )\n if response.error:\n print(\n f\"Error: couldn't cancel '{self.run_name}' for {self.run_time_str}. \"\n \"You should use tronview to check on it.\"\n )\n else:\n print(f\"Backfill job run '{self.run_name}' for {self.run_time_str} cancelled\")\n self.run_state = ActionRun.CANCELLED\n return True\n else:\n # accounts for the case where the job was created, but this coroutine\n # is cancelled before the name (and id) is returned to us\n print(\n f\"Warning: attempted to cancel backfill for {self.run_time_str}, but we \"\n \"don't know if it was created initially. You should use tronview \"\n \"to check.\"\n )\n return False\n\n\nasync def run_backfill_for_date_range(\n server: str,\n job_name: str,\n dates: list[datetime.datetime],\n max_parallel: int = DEFAULT_MAX_PARALLEL_RUNS,\n ignore_errors: bool = True,\n) -> list[BackfillRun]:\n \"\"\"Creates and watches job runs over a range of dates for a given job. At\n most, max_parallel runs can run in parallel to prevent resource exhaustion.\n \"\"\"\n loop = asyncio.get_event_loop()\n\n # Trigger authentication before submitting all async jobs, so auth tokens\n # are cached and won't prompt the user in the individual API calls.\n # We pass `no_cache` to ensure a new refresh token is generated and stored in memory.\n if os.getenv(\"TRONCTL_API_AUTH\"):\n get_auth_token(no_cache=True)\n\n tron_client = client.Client(server, user_attribution=True)\n url_index = tron_client.index()\n\n # check job_name identifies a valid tron object\n job_id = client.get_object_type_from_identifier(url_index, job_name)\n # check job_name identifies a job\n if job_id.type != client.TronObjectType.job:\n raise ValueError(f\"'{job_name}' is a {job_id.type.lower()}, not a job\")\n\n backfill_runs = [BackfillRun(tron_client, job_id, run_time) for run_time in dates]\n running: set[asyncio.Future] = set()\n finished_cnt = 0\n all_successful = True\n\n # `current_task()` will always return a task here, but we need to account\n # for the None case for mypy\n current_task = asyncio.current_task()\n if current_task:\n loop.add_signal_handler(signal.SIGINT, current_task.cancel)\n try:\n while finished_cnt < len(dates):\n # start more runs if we still have some and tha parallel limit is not yet reached\n while finished_cnt + len(running) < len(dates) and len(running) < max_parallel:\n next_run = backfill_runs[finished_cnt + len(running)]\n running.add(asyncio.ensure_future(next_run.run_until_completion()))\n\n just_finished, running = await asyncio.wait(running, return_when=asyncio.FIRST_COMPLETED)\n for task in just_finished:\n finished_cnt += 1\n all_successful &= task.result() in BackfillRun.SUCCESS_STATES\n\n if not ignore_errors and not all_successful:\n print(\"Error: encountered failing job run; cancelling all in-progress runs and exiting.\")\n for task in running:\n task.cancel() # cancel running async tasks\n await task # wait until it is done cancelling\n break\n\n except asyncio.CancelledError: # caused by sigint handler\n print(\"Error: SIGINT detected; aborting all in-progress runs and exiting\")\n for task in running:\n task.cancel()\n await task\n return backfill_runs\n\n\nclass DisplayBackfillRuns(display.TableDisplay):\n\n columns = [\"Date\", \"Job Run Name\", \"Final State\"]\n fields = [\"run_time\", \"run_name\", \"run_state\"]\n widths = [15, 60, 15]\n title = \"Backfills Job Runs\"\n resize_fields = {\"run_time\", \"run_name\", \"run_state\"}\n header_color = \"hgray\"\n\n\ndef print_backfill_runs_table(runs: list[BackfillRun]) -> None:\n \"\"\"Prints backfill runs in a table\"\"\"\n with display.Color.enable():\n table = DisplayBackfillRuns().format(\n [\n dict(run_time=r.run_time.date().isoformat(), run_name=(r.run_name or \"n/a\"), run_state=r.run_state)\n for r in runs\n ]\n )\n print(table)\n" }, { "path": "tron/commands/client.py", "content": "\"\"\"\nA command line http client used by tronview, tronctl, and tronfig\n\"\"\"\nimport json\nimport logging\nimport os\nimport urllib.error\nimport urllib.parse\nimport urllib.request\nfrom collections import namedtuple\n\nimport tron\nfrom tron.commands.authentication import get_auth_token\nfrom tron.config.schema import MASTER_NAMESPACE\n\n\nlog = logging.getLogger(__name__)\n\nUSER_AGENT = f\"Tron Command/{tron.__version__} +http://github.com/Yelp/Tron\"\nDECODE_ERROR = \"DECODE_ERROR\"\nURL_ERROR = \"URL_ERROR\"\n\n\nclass RequestError(ValueError):\n \"\"\"Raised when the request to tron API fails.\"\"\"\n\n\nResponse = namedtuple(\"Response\", \"error msg content\")\n\ndefault_headers = {\n \"User-Agent\": USER_AGENT,\n}\n\n\ndef build_url_request(uri, data, headers=None, method=None):\n headers = headers or default_headers\n enc_data = urllib.parse.urlencode(data).encode() if data else None\n # Currently implementing auth only for management actions (i.e. POST requests)\n if os.getenv(\"TRONCTL_API_AUTH\") and (data or (method and method.upper() == \"POST\")):\n token = get_auth_token()\n if token:\n headers[\"Authorization\"] = f\"Bearer {token}\"\n return urllib.request.Request(uri, enc_data, headers=headers, method=method)\n\n\ndef load_response_content(http_response):\n encoding = http_response.headers.get_content_charset()\n if encoding is None:\n encoding = \"utf8\"\n content = http_response.read().decode(encoding)\n try:\n return Response(None, None, json.loads(content))\n except ValueError as e:\n log.error(\"Failed to decode response: %s, %s\", e, content)\n return Response(DECODE_ERROR, str(e), content)\n\n\ndef build_http_error_response(exc):\n content = exc.read() if hasattr(exc, \"read\") else None\n if content:\n encoding = exc.headers.get_content_charset()\n if encoding is None:\n encoding = \"utf8\"\n content = content.decode(encoding)\n try:\n content = json.loads(content)\n content = content[\"error\"]\n except ValueError:\n log.warning(\n f\"Incorrectly formatted error response: {content}\",\n )\n return Response(exc.code, exc.msg, content)\n\n\ndef request(uri, data=None, headers=None, method=None, user_attribution=False):\n log.info(\"Request to %s with %s\", uri, data)\n headers = headers or default_headers\n if user_attribution:\n headers = ensure_user_attribution(headers)\n request = build_url_request(uri, data, headers=headers, method=method)\n try:\n response = urllib.request.urlopen(request)\n except urllib.error.HTTPError as e:\n log.error(\"Received error response: %s\" % e)\n return build_http_error_response(e)\n except urllib.error.URLError as e:\n log.error(\"Received error response: %s\" % e)\n return Response(URL_ERROR, e.reason, None)\n\n return load_response_content(response)\n\n\ndef build_get_url(url, data=None):\n if data:\n query_str = urllib.parse.urlencode(sorted(data.items()))\n return f\"{url}?{query_str}\"\n else:\n return url\n\n\ndef ensure_user_attribution(headers: dict[str, str]) -> dict[str, str]:\n headers = headers.copy()\n if \"User-Agent\" not in headers:\n headers[\"User-Agent\"] = USER_AGENT\n headers[\"User-Agent\"] += f' ({os.environ.get(\"USER\", \"anonymous\")})'\n return headers\n\n\nclass Client:\n \"\"\"An HTTP client used to issue commands to the Tron API.\"\"\"\n\n def __init__(self, url_base, cluster_name=None, user_attribution=False):\n \"\"\"Create a new client.\n url_base - A url with a schema, hostname and port\n \"\"\"\n self.url_base = url_base\n self.cluster_name = cluster_name\n self.headers = default_headers\n if user_attribution:\n self.headers = ensure_user_attribution(self.headers)\n\n def status(self):\n return self.http_get(\"/api/status\")\n\n def metrics(self):\n return self.http_get(\"/api/metrics\")\n\n def config(\n self,\n config_name,\n config_data=None,\n config_hash=None,\n check=False,\n ):\n \"\"\"Retrieve or update the configuration.\"\"\"\n if config_data is not None:\n data_check = 1 if check else 0\n request_data = dict(\n config=config_data,\n name=config_name,\n hash=config_hash,\n check=data_check,\n )\n return self.request(\"/api/config\", request_data)\n request_data = dict(name=config_name)\n return self.http_get(\"/api/config\", request_data)\n\n def home(self):\n return self.http_get(\"/api/\")\n\n index = home\n\n def get_url(self, identifier):\n return get_object_type_from_identifier(self.index(), identifier).url\n\n def jobs(\n self,\n include_job_runs=False,\n include_action_runs=False,\n include_action_graph=True,\n include_node_pool=True,\n ):\n params = {\n \"include_job_runs\": int(include_job_runs),\n \"include_action_runs\": int(include_action_runs),\n \"include_action_graph\": int(include_action_graph),\n \"include_node_pool\": int(include_node_pool),\n }\n return self.http_get(\"/api/jobs\", params).get(\"jobs\")\n\n def job(self, job_url, include_action_runs=False, count=0):\n params = {\n \"include_action_runs\": int(include_action_runs),\n \"num_runs\": count,\n }\n return self.http_get(job_url, params)\n\n def job_runs(self, url, include_runs=True, include_graph=False):\n params = {\n \"include_action_runs\": int(include_runs),\n \"include_action_graph\": int(include_graph),\n }\n return self.http_get(url, params)\n\n def action_runs(self, action_run_url, num_lines=0):\n params = {\n \"num_lines\": num_lines,\n \"include_stdout\": 1,\n \"include_stderr\": 1,\n }\n return self.http_get(action_run_url, params)\n\n def http_get(self, url, data=None):\n return self.request(build_get_url(url, data))\n\n def request(self, url, data=None):\n uri = urllib.parse.urljoin(self.url_base, url)\n response = request(uri, data, headers=self.headers)\n if response.error:\n if response.content:\n raise RequestError(response.content)\n else:\n raise RequestError(f\"{response.error} {response.msg}\")\n return response.content\n\n\ndef build_api_url(resource, identifier_parts):\n return \"/api/{}/{}\".format(resource, \"/\".join(identifier_parts))\n\n\ndef split_identifier(identifier):\n return identifier.rsplit(\".\", identifier.count(\".\") - 1)\n\n\ndef get_job_url(identifier):\n return build_api_url(\"jobs\", split_identifier(identifier))\n\n\nclass TronObjectType:\n \"\"\"Constants to identify a Tron object type.\"\"\"\n\n job = \"JOB\"\n job_run = \"JOB_RUN\"\n action_run = \"ACTION_RUN\"\n\n url_builders = {\n \"jobs\": get_job_url,\n }\n\n groups = {\n \"jobs\": [job, job_run, action_run],\n }\n\n\nTronObjectIdentifier = namedtuple(\"TronObjectIdentifier\", \"type url\")\n\nIdentifierParts = namedtuple(\"IdentifierParts\", \"name full_id length\")\n\n\ndef first(seq):\n for item in filter(None, seq):\n return item\n\n\ndef get_object_type_from_identifier(url_index, identifier):\n \"\"\"Given a string identifier, return a TronObjectIdentifier.\"\"\"\n name_mapping = {\n \"jobs\": set(url_index[\"jobs\"]),\n }\n\n def get_name_parts(identifier, namespace=None):\n if namespace:\n identifier = f\"{namespace}.{identifier}\"\n\n name_elements = identifier.split(\".\")\n name = \".\".join(name_elements[:2])\n length = len(name_elements) - 2\n return IdentifierParts(name, identifier, length)\n\n def find_by_type(id_parts, index_name):\n url_type_index = name_mapping[index_name]\n if id_parts.name in url_type_index:\n tron_type = TronObjectType.groups[index_name][id_parts.length]\n url = TronObjectType.url_builders[index_name](id_parts.full_id)\n return TronObjectIdentifier(tron_type, url)\n\n def find_by_name(name, namespace=None):\n id = get_name_parts(name, namespace)\n return find_by_type(id, \"jobs\")\n\n namespaces = [None, MASTER_NAMESPACE] + url_index[\"namespaces\"]\n id_obj = first(find_by_name(identifier, name) for name in namespaces)\n if id_obj:\n return id_obj\n\n raise ValueError(\"Unknown job identifier: %s\" % identifier)\n" }, { "path": "tron/commands/cmd_utils.py", "content": "\"\"\"\nCommon code for command line utilities (see bin/)\n\"\"\"\nimport argparse\nimport difflib\nimport logging\nimport os\nimport sys\n\nimport tron\nfrom tron import yaml\n\nlog = logging.getLogger(\"tron.commands\")\n\n\nclass ExitCode:\n \"\"\"Enumeration of exit status codes.\"\"\"\n\n success = 0\n fail = 1\n\n\nGLOBAL_CONFIG_FILE_NAME = (\n os.environ.get(\n \"TRON_CONFIG\",\n )\n or \"/etc/tron/tron.yaml\"\n)\nCONFIG_FILE_NAME = os.path.expanduser(\"~/.tron\")\n\nDEFAULT_HOST = \"localhost\"\nDEFAULT_PORT = 8089\n\nDEFAULT_CONFIG = {\n \"server\": \"http://%s:%d\" % (DEFAULT_HOST, DEFAULT_PORT),\n \"display_color\": False,\n \"cluster_name\": \"Unnamed Cluster\",\n}\n\nTAB_COMPLETE_FILE = \"/var/cache/tron_tab_completions\"\n\nCOLOR_RED = \"\\033[31m\"\nCOLOR_YELLOW = \"\\033[33m\"\nCOLOR_DEFAULT = \"\\033[0m\"\n\nopener = open\n\n\ndef get_default_server():\n return DEFAULT_CONFIG[\"server\"]\n\n\ndef filter_jobs_actions_runs(prefix, inputs):\n dots = prefix.count(\".\")\n if prefix == \"\":\n # If the user hasn't begun to type anything, we need to get them started with all jobs\n return [i for i in inputs if i.count(\".\") == 1]\n elif dots == 0:\n # If the user hasn't completed a job, we need to get them started with all jobs\n # that start with what they have\n return [i for i in inputs if i.count(\".\") == 1 and i.startswith(prefix)]\n elif prefix in inputs:\n # If what a user typed is exactly what is already in a suggestion, then we need to give them\n # Even more suggestions (+1)\n return [i for i in inputs if i.startswith(prefix) and (i.count(\".\") == dots or i.count(\".\") == dots + 1)]\n else:\n # Otherwise we only want to scope our suggestions to those that are on the same \"level\"\n # which in string form means they have the same number of dots\n return [i for i in inputs if i.startswith(prefix) and i.count(\".\") == dots]\n\n\ndef tron_jobs_completer(prefix, **kwargs):\n if os.path.isfile(TAB_COMPLETE_FILE):\n with opener(TAB_COMPLETE_FILE, \"r\") as f:\n jobs = f.readlines()\n return filter_jobs_actions_runs(\n prefix=prefix,\n inputs=[job.strip(\"\\n\\r\") for job in jobs],\n )\n else:\n # this import is here to avoid an annoying circular dependency\n from tron.commands.client import Client\n\n if \"client\" not in kwargs:\n client = Client(get_default_server())\n else:\n client = kwargs[\"client\"]\n return filter_jobs_actions_runs(\n prefix=prefix,\n inputs=[job[\"name\"] for job in client.jobs()],\n )\n\n\ndef build_option_parser(usage=None, epilog=None):\n parser = argparse.ArgumentParser(\n usage=usage,\n epilog=epilog,\n formatter_class=argparse.RawDescriptionHelpFormatter,\n )\n parser.add_argument(\n \"--version\",\n action=\"version\",\n version=f\"{parser.prog} {tron.__version__}\",\n )\n\n parser.add_argument(\n \"-v\",\n \"--verbose\",\n action=\"count\",\n help=\"Verbose logging\",\n default=None,\n )\n parser.add_argument(\n \"--server\",\n default=None,\n help=\"Url including scheme, host and port, Default: %(default)s\",\n )\n parser.add_argument(\n \"--cluster_name\",\n default=None,\n help=\"Human friendly tron cluster name\",\n )\n parser.add_argument(\n \"-s\",\n \"--save\",\n action=\"store_true\",\n dest=\"save_config\",\n help=\"Save options used on this job for next time.\",\n )\n\n return parser\n\n\ndef get_client_config():\n config_file_list = [CONFIG_FILE_NAME, GLOBAL_CONFIG_FILE_NAME]\n for config_file in config_file_list:\n filename = os.path.expanduser(config_file)\n if os.access(filename, os.R_OK):\n config = read_config(filename)\n if config:\n return config\n\n log.debug(\"Could not find a config in: %s.\" % \", \".join(config_file_list))\n return {}\n\n\ndef load_config(options):\n \"\"\"Attempt to load a user specific configuration or a global config file\n and set any unset options based on values from the config. Finally fallback\n to DEFAULT_CONFIG for those settings.\n\n Also save back options to the config if options.save_config is True.\n \"\"\"\n config = get_client_config()\n\n for opt_name in DEFAULT_CONFIG.keys():\n if not hasattr(options, opt_name):\n continue\n\n if getattr(options, opt_name) is not None:\n continue\n\n default_value = DEFAULT_CONFIG[opt_name]\n setattr(options, opt_name, config.get(opt_name, default_value))\n\n if options.save_config:\n save_config(options)\n\n\ndef read_config(filename=CONFIG_FILE_NAME):\n try:\n with opener(filename, \"r\") as config_file:\n return yaml.load(config_file)\n except OSError:\n log.info(\"Failed to read config file: %s\" % CONFIG_FILE_NAME)\n return {}\n\n\ndef write_config(config):\n with open(CONFIG_FILE_NAME, \"w\") as config_file:\n yaml.dump(config, config_file)\n\n\ndef save_config(options):\n config = read_config()\n for opt_name in DEFAULT_CONFIG.keys():\n if not hasattr(options, opt_name):\n continue\n config[opt_name] = getattr(options, opt_name)\n write_config(config)\n\n\ndef setup_logging(options: argparse.Namespace) -> int:\n if options.verbose is None:\n level = logging.ERROR\n elif options.verbose == 1:\n level = logging.WARNING\n elif options.verbose == 2:\n level = logging.INFO\n else:\n level = logging.NOTSET\n\n logging.basicConfig(\n level=level,\n format=\"%(name)s %(levelname)s %(message)s\",\n stream=sys.stdout,\n )\n\n return level\n\n\ndef suggest_possibilities(word, possibilities, max_suggestions=6):\n suggestions = difflib.get_close_matches(\n word=word,\n possibilities=possibilities,\n n=max_suggestions,\n )\n if len(suggestions) == 1:\n return f\"\\nDid you mean: {suggestions[0]}?\"\n elif len(suggestions) >= 1:\n return f\"\\nDid you mean one of: {', '.join(suggestions)}?\"\n else:\n return \"\"\n\n\ndef warning_output(text: str, color: str = COLOR_RED) -> str:\n \"\"\"Return the passed-in string colored in red (by default). Suitable for warning messages.\"\"\"\n return f\"{color}{text}{COLOR_DEFAULT}\"\n" }, { "path": "tron/commands/display.py", "content": "\"\"\"\nFormat and color output for tron commands.\n\"\"\"\nimport contextlib\nfrom collections.abc import Callable\nfrom collections.abc import Collection\nfrom functools import partial\nfrom operator import itemgetter\n\nfrom tron.core import actionrun\nfrom tron.core import job\nfrom tron.utils import exitcode\nfrom tron.utils import maybe_encode\n\n\nclass Color:\n\n enabled = None\n colors = {\n \"gray\": \"\\033[90m\",\n \"red\": \"\\033[91m\",\n \"green\": \"\\033[92m\",\n \"yellow\": \"\\033[93m\",\n \"blue\": \"\\033[94m\",\n \"purple\": \"\\033[95m\",\n \"cyan\": \"\\033[96m\",\n \"white\": \"\\033[99m\",\n # h is for highlighted\n \"hgray\": \"\\033[100m\",\n \"hred\": \"\\033[101m\",\n \"hgreen\": \"\\033[102m\",\n \"hyellow\": \"\\033[103m\",\n \"hblue\": \"\\033[104m\",\n \"hcyan\": \"\\033[106m\",\n \"end\": \"\\033[0m\",\n }\n\n @classmethod\n @contextlib.contextmanager\n def enable(cls):\n old_val = cls.enabled\n try:\n cls.enabled = True\n yield\n finally:\n cls.enabled = old_val\n\n @classmethod\n def set(cls, color_name, text):\n if not cls.enabled or not color_name:\n return text\n return \"{}{}{}\".format(\n cls.colors[color_name.lower()],\n text,\n cls.colors[\"end\"],\n )\n\n @classmethod\n def toggle(cls, enable):\n cls.enabled = enable\n\n\nclass TableDisplay:\n \"\"\"Base class for displaying columns of data. This class takes a list\n of dict objects and formats it so that it displays properly in fixed width\n columns. Overlap is truncated.\n\n This class provides many hooks for customizing the output, including:\n - sorting of rows\n - building composite values from more then one field\n - custom formatting of a columns values\n - adding additional data after each row\n - coloring of header, columns, or rows\n\n The default output is:\n\n Banner\n Header\n Row\n (optional post row)\n Row\n (optional post row)\n ...\n Footer\n \"\"\"\n\n columns: list[str] = []\n fields: list[str] = []\n widths: list[int] = []\n colors: dict[str, Callable[[str], str]] = {}\n title: str | None = None\n resize_fields: Collection[str] = set()\n reversed = False\n\n header_color = \"hgray\"\n\n def __init__(self, sort_index=0):\n self.out = []\n self.sort_index = sort_index\n\n def banner(self):\n if not self.title:\n return\n title = self.title.capitalize()\n self.out.append(\"\\n%s:\" % title)\n if not self.rows():\n self.out.append(\"No %s\\n\" % title)\n\n def header(self):\n row = [label.ljust(self.get_field_width(i)) for i, label in enumerate(self.columns)]\n self.out.append(Color.set(self.header_color, \"\".join(row)))\n\n def footer(self):\n pass\n\n def color(self, col, field):\n return None\n\n def sorted_fields(self, values):\n return [values[name] for name in self.fields]\n\n def format_row(self, fields):\n row = [\n Color.set(self.color(i, value), self.trim_value(i, value))\n for i, value in enumerate(self.sorted_fields(fields))\n ]\n return Color.set(self.row_color(fields), \"\".join(row))\n\n def get_field_width(self, field_idx):\n return self.widths[field_idx]\n\n def trim_value(self, field_idx, value):\n length = self.get_field_width(field_idx)\n value = self.format_value(field_idx, value)\n if len(value) > length:\n return (value[: length - 3] + \"...\").ljust(length)\n return value.ljust(length)\n\n def format_value(self, field_idx, value):\n return str(value)\n\n def output(self):\n out = \"\\n\".join(self.out)\n self.out = []\n return out\n\n def post_row(self, row):\n pass\n\n def row_color(self, row):\n return None\n\n def rows(self):\n return sorted(\n self.data,\n key=itemgetter(self.fields[self.sort_index]),\n reverse=self.reversed,\n )\n\n def store_data(self, data):\n self.data = data\n\n def update_column_widths(self):\n \"\"\"Update column widths to fit the data.\"\"\"\n for field_idx, field in enumerate(self.fields):\n if field in self.resize_fields:\n self.widths[field_idx] = self.calculate_width(field_idx)\n\n def calculate_width(self, field_idx):\n default_width = self.widths[field_idx]\n column = [self.format_value(field_idx, row[self.fields[field_idx]]) for row in self.data]\n if not column:\n return default_width\n max_value_width = max(len(value) for value in column)\n return max(max_value_width + 1, default_width)\n\n def format(self, data):\n self.store_data(data)\n self.update_column_widths()\n self.banner()\n\n if not self.rows():\n return self.output()\n\n self.header()\n for row in self.rows():\n self.out.append(self.format_row(row))\n self.post_row(row)\n\n self.footer()\n return self.output()\n\n\ndef add_color_for_state(state):\n if state == actionrun.ActionRun.FAILED:\n return Color.set(\"red\", state)\n if state in {\n actionrun.ActionRun.RUNNING,\n actionrun.ActionRun.SUCCEEDED,\n job.Job.STATUS_ENABLED,\n }:\n return Color.set(\"green\", state)\n if state in {job.Job.STATUS_DISABLED}:\n return Color.set(\"blue\", state)\n return state\n\n\ndef format_fields(display_obj, content):\n \"\"\"Format fields with some color.\"\"\"\n\n def add_color(field, field_value):\n if field not in display_obj.colors:\n return field_value\n return display_obj.colors[field](field_value)\n\n def format_field(field):\n formatter = field_display_mapping.get(field, lambda f, _: f)\n return formatter(content.get(field), content)\n\n def build_field(label, field):\n return f\"{label:<20}: {add_color(field, format_field(field))}\"\n\n return \"\\n\".join(build_field(*item) for item in display_obj.detail_labels)\n\n\ndef format_job_details(job_content):\n details = format_fields(DisplayJobs, job_content)\n job_runs = DisplayJobRuns().format(job_content[\"runs\"])\n actions = \"\\n\\nList of Actions:\\n%s\" % \"\\n\".join(\n job_content[\"action_names\"],\n )\n return details + actions + \"\\n\" + job_runs\n\n\ndef format_action_run_details(content, stdout=True, stderr=True):\n out = [\"Requirements:\"] + content[\"requirements\"] + [\"\"]\n if stdout:\n out.append(\"Stdout:\\n%s\\n\" % \"\\n\".join(content[\"stdout\"]))\n\n if stderr:\n out.append(\"Stderr:\\n%s\\n\" % \"\\n\".join(content[\"stderr\"]))\n\n details = format_fields(DisplayActionRuns, content)\n return details + \"\\n\" + \"\\n\".join(out)\n\n\nclass DisplayJobRuns(TableDisplay):\n \"\"\"Format Job runs.\"\"\"\n\n columns = [\"Run ID\", \"State\", \"Node\", \"Scheduled Time\"]\n fields = [\"run_num\", \"state\", \"node\", \"run_time\"]\n widths = [10, 12, 30, 25]\n title = \"job runs\"\n reversed = True\n\n detail_labels = [\n (\"Job Run\", \"id\"),\n (\"State\", \"state\"),\n (\"Node\", \"node\"),\n (\"Scheduled time\", \"run_time\"),\n (\"Start time\", \"start_time\"),\n (\"End time\", \"end_time\"),\n (\"Manual run\", \"manual\"),\n ]\n\n colors = {\n \"id\": partial(Color.set, \"yellow\"),\n \"state\": add_color_for_state,\n \"manual\": lambda value: Color.set(\"cyan\" if value else None, value),\n }\n\n def format_value(self, field_idx, value):\n if self.fields[field_idx] == \"run_num\":\n value = \".\" + str(value)\n\n if self.fields[field_idx] == \"scheduled_time\":\n value = value or \"-\"\n\n if self.fields[field_idx] == \"node\":\n value = display_node(value)\n\n return super().format_value(field_idx, value)\n\n def row_color(self, fields):\n return \"red\" if fields[\"state\"] == \"FAIL\" else \"white\"\n\n def post_row(self, row):\n start = row[\"start_time\"] or \"-\"\n end = row[\"end_time\"] or \"-\"\n duration = row[\"duration\"][:-7] if row[\"duration\"] else \"-\"\n\n row_data = \"{}Start: {} End: {} ({})\".format(\n \" \" * self.widths[0],\n start,\n end,\n duration,\n )\n self.out.append(Color.set(\"gray\", row_data))\n\n\nclass DisplayJobs(TableDisplay):\n\n columns = [\"Name\", \"State\", \"Scheduler\", \"Last Success\"]\n fields = [\"name\", \"status\", \"scheduler\", \"last_success\"]\n widths = [50, 10, 20, 22]\n title = \"jobs\"\n resize_fields = [\"name\"]\n\n detail_labels = [\n (\"Job\", \"name\"),\n (\"State\", \"status\"),\n (\"Scheduler\", \"scheduler\"),\n (\"Max runtime\", \"max_runtime\"),\n (\"Node Pool\", \"node_pool\"),\n (\"Run on all nodes\", \"all_nodes\"),\n (\"Allow overlapping\", \"allow_overlap\"),\n (\"Queue overlapping\", \"queueing\"),\n ]\n\n colors = {\n \"name\": partial(Color.set, \"yellow\"),\n \"status\": add_color_for_state,\n }\n\n def format_value(self, field_idx, value):\n if self.fields[field_idx] == \"scheduler\":\n value = display_scheduler(value)\n\n return super().format_value(field_idx, value)\n\n\nclass DisplayActionRuns(TableDisplay):\n\n columns = [\"Action\", \"State\", \"Start Time\", \"End Time\", \"Duration\"]\n fields = [\"id\", \"state\", \"start_time\", \"end_time\", \"duration\"]\n widths = [40, 12, 22, 22, 10]\n title = \"actions\"\n resize_fields = [\"id\"]\n\n detail_labels = [\n (\"Action Run\", \"id\"),\n (\"State\", \"state_delayed\"),\n (\"Node\", \"node\"),\n (\"Last run command\", \"command\"),\n (\"Original raw command\", \"original_command\"),\n (\"Config command\", \"raw_command\"),\n (\"Start time\", \"start_time\"),\n (\"End time\", \"end_time\"),\n (\"Final exit status\", \"exit_status\"),\n (\"Exit statuses\", \"exit_statuses\"),\n (\"Waits for triggers\", \"triggered_by\"),\n (\"Publishes triggers\", \"trigger_downstreams\"),\n ]\n\n colors = {\n \"id\": partial(Color.set, \"yellow\"),\n \"state\": add_color_for_state,\n \"command\": partial(Color.set, \"gray\"),\n }\n\n def __init__(self):\n # Action runs need to be storted by start time, which is index 2\n super().__init__(sort_index=2)\n\n def banner(self):\n self.out.append(format_fields(DisplayJobRuns, self.job_run))\n super().banner()\n\n def format_value(self, field_idx, value):\n if self.fields[field_idx] == \"id\":\n value = \".\" + value.rsplit(\".\", 1)[-1]\n if self.fields[field_idx] in (\"start_time\", \"end_time\"):\n value = value or \"-\"\n if self.fields[field_idx] == \"duration\":\n # Strip microseconds\n value = value[:-7] if value else \"-\"\n\n return super().format_value(field_idx, value)\n\n def row_color(self, fields):\n return \"red\" if fields[\"state\"] == \"FAIL\" else \"white\"\n\n def store_data(self, data):\n self.data = data[\"runs\"]\n self.job_run = data\n\n def rows(self):\n # Action runs need a sort order that sorts by date\n # and that can handle situations where it is None, or\n # othere weird things, so we str()\n return sorted(\n self.data,\n key=lambda x: str(x[self.fields[self.sort_index]]),\n reverse=self.reversed,\n )\n\n\ndef display_node(source, _=None):\n if not source:\n return \"\"\n return \"{}@{}\".format(source[\"username\"], source[\"hostname\"])\n\n\ndef display_node_pool(source, _=None):\n if not source:\n return \"\"\n return \"%s (%d node(s))\" % (source[\"name\"], len(source[\"nodes\"]))\n\n\ndef display_scheduler(source, _=None):\n if not source:\n return \"\"\n return \"{} {}{}\".format(source[\"type\"], source[\"value\"], source[\"jitter\"])\n\n\ndef display_state_delayed(_, obj):\n state = obj[\"state\"]\n in_delay = obj[\"in_delay\"]\n if in_delay:\n return f\"{state} (retry delayed for {int(in_delay)}s)\"\n else:\n return state\n\n\nfield_display_mapping = {\n \"node\": display_node,\n \"node_pool\": display_node_pool,\n \"scheduler\": display_scheduler,\n \"state_delayed\": display_state_delayed,\n \"exit_status\": lambda v, _: exitcode.EXIT_REASONS.get(v, v),\n}\n\n\ndef view_with_less(content, color=True):\n \"\"\"Send `content` through less.\"\"\"\n import subprocess\n\n cmd = [\"less\"]\n if color:\n cmd.append(\"-r\")\n\n less_proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)\n less_proc.stdin.write(\n maybe_encode(content)\n ) # TODO: TRON-2293 maybe_encode is a relic of Python2->Python3 migration. Remove it.\n less_proc.stdin.close()\n less_proc.wait()\n" }, { "path": "tron/commands/retry.py", "content": "import argparse\nimport asyncio\nimport datetime\nimport functools\nimport random\nfrom urllib.parse import urljoin\n\nimport pytimeparse # type:ignore\n\nfrom tron.commands import client\nfrom tron.commands import display\nfrom tron.commands.backfill import BackfillRun\n\n\nDEFAULT_POLLING_INTERVAL_S = 10\n\n\ndef parse_deps_timeout(duration: str) -> int:\n if duration == \"infinity\":\n return RetryAction.WAIT_FOREVER\n elif duration.isnumeric():\n seconds = int(duration)\n else:\n seconds = pytimeparse.parse(duration)\n if seconds is None:\n raise argparse.ArgumentTypeError(\n f\"'{duration}' is not a valid duration. Must be either number of seconds or pytimeparse-parsable string.\"\n )\n if seconds < 0:\n raise argparse.ArgumentTypeError(f\"'{duration}' must not be negative\")\n return seconds\n\n\nclass RetryAction:\n NO_TIMEOUT = 0\n WAIT_FOREVER = -1\n\n RETRY_NOT_ISSUED = None\n RETRY_SUCCESS = True\n RETRY_FAIL = False\n\n def __init__(\n self,\n tron_client: client.Client,\n full_action_name: str,\n use_latest_command: bool = False,\n ):\n self.tron_client = tron_client\n self.retry_params = dict(command=\"retry\", use_latest_command=int(use_latest_command))\n\n self.full_action_name = full_action_name\n self.action_run_id = self._validate_action_name(full_action_name)\n self.job_run_id = client.get_object_type_from_identifier(self.tron_client.index(), self.job_run_name)\n\n self._required_action_indices = self._get_required_action_indices()\n self._elapsed = datetime.timedelta(seconds=0)\n self._triggers_done = False\n self._required_actions_done = False\n self._retry_request_result: bool | None = RetryAction.RETRY_NOT_ISSUED\n\n @property\n def job_run_name(self) -> str:\n return self.full_action_name.rsplit(\".\", 1)[0]\n\n @property\n def action_name(self) -> str:\n return self.full_action_name.rsplit(\".\", 1)[1]\n\n @property\n def status(self) -> str:\n if not self._triggers_done:\n return \"Upstream triggers not all published\"\n elif not self._required_actions_done:\n return \"Required actions not all successfully completed\"\n elif self._retry_request_result == RetryAction.RETRY_NOT_ISSUED:\n return \"Retry request not issued, but dependencies done\"\n elif self._retry_request_result == RetryAction.RETRY_SUCCESS:\n return \"Retry request issued successfully\"\n else:\n return \"Failed to issue retry request\"\n\n @property\n def succeeded(self) -> bool:\n return bool(self._retry_request_result)\n\n def _validate_action_name(self, full_action_name: str) -> client.TronObjectIdentifier:\n action_run_id: client.TronObjectIdentifier = client.get_object_type_from_identifier(\n self.tron_client.index(), full_action_name\n )\n if action_run_id.type != client.TronObjectType.action_run:\n raise ValueError(f\"'{full_action_name}' is a {action_run_id.type.lower()}, not an action\")\n self.tron_client.action_runs(action_run_id.url, num_lines=0) # verify action exists\n return action_run_id\n\n def _get_required_action_indices(self) -> dict[str, int]:\n job_run = self.tron_client.job_runs(self.job_run_id.url)\n required_actions = set()\n action_indices = {}\n\n for i, action_run in enumerate(job_run[\"runs\"]):\n if action_run[\"action_name\"] == self.action_name:\n required_actions = set(action_run[\"requirements\"])\n action_indices[action_run[\"action_name\"]] = i\n\n return {action_name: i for action_name, i in action_indices.items() if action_name in required_actions}\n\n def _log(self, msg: str) -> None:\n print(f\"[{self._elapsed}] {self.full_action_name}: {msg}\")\n\n async def can_retry(self) -> bool:\n if not self._triggers_done:\n triggers = await self.check_trigger_statuses()\n self._triggers_done = all(triggers.values())\n if self._triggers_done:\n if len(triggers) > 0:\n self._log(\"All upstream triggers published\")\n else:\n remaining_triggers = [trigger for trigger, is_done in triggers.items() if not is_done]\n self._log(f\"Upstream triggers not yet published: {remaining_triggers}\")\n if not self._required_actions_done:\n required_actions = await self.check_required_actions_statuses()\n self._required_actions_done = all(required_actions.values())\n if self._required_actions_done:\n if len(required_actions) > 0:\n self._log(\"All required actions finished\")\n else:\n remaining_required_actions = [action for action, is_done in required_actions.items() if not is_done]\n self._log(f\"Required actions not yet succeeded: {remaining_required_actions}\")\n return self._triggers_done and self._required_actions_done\n\n async def check_trigger_statuses(self) -> dict[str, bool]:\n action_run = await asyncio.get_event_loop().run_in_executor(\n None,\n functools.partial(\n self.tron_client.action_runs,\n self.action_run_id.url,\n num_lines=0,\n ),\n )\n # from tron.api.adapter:ActionRunAdapter.get_triggered_by:\n # triggered_by is a single string with this format:\n # {trigger_1} (done), {trigger_2}, etc.\n # where trigger_1 has been published, and trigger_2 is still waiting\n trigger_states = {}\n for trigger_and_state in action_run[\"triggered_by\"].split(\", \"):\n if trigger_and_state:\n trigger, *maybe_state = trigger_and_state.split(\" \")\n # if len(parts) == 2, then parts is [{trigger}, \"(done)\"]\n # else, parts is [{trigger}]\n trigger_states[trigger] = len(maybe_state) == 1\n return trigger_states\n\n async def check_required_actions_statuses(self) -> dict[str, bool]:\n action_runs = (\n await asyncio.get_event_loop().run_in_executor(\n None,\n self.tron_client.job_runs,\n self.job_run_id.url,\n )\n )[\"runs\"]\n return {\n action_runs[i][\"action_name\"]: action_runs[i][\"state\"] in BackfillRun.SUCCESS_STATES\n for i in self._required_action_indices.values()\n }\n\n async def wait_and_retry(\n self,\n deps_timeout_s: int = 0,\n poll_interval_s: int = DEFAULT_POLLING_INTERVAL_S,\n jitter: bool = True,\n ) -> bool:\n\n if deps_timeout_s != RetryAction.NO_TIMEOUT and jitter:\n init_delay_s = random.randint(1, min(deps_timeout_s, poll_interval_s)) - 1\n self._elapsed += datetime.timedelta(seconds=init_delay_s)\n await asyncio.sleep(init_delay_s)\n\n if await self.wait_for_deps(deps_timeout_s=deps_timeout_s, poll_interval_s=poll_interval_s):\n return await self.issue_retry()\n else:\n deps_timeout_td = datetime.timedelta(seconds=deps_timeout_s)\n msg = \"Action will not be retried.\"\n if deps_timeout_s != RetryAction.NO_TIMEOUT:\n msg = f\"Not all dependencies completed after waiting for {deps_timeout_td}. \" + msg\n self._log(msg)\n return False\n\n async def wait_for_deps(\n self,\n deps_timeout_s: int = 0,\n poll_interval_s: int = DEFAULT_POLLING_INTERVAL_S,\n ) -> bool:\n \"\"\"Wait for all upstream dependencies to finished up to a timeout. Once the\n timeout has expired, one final check is always conducted.\n\n Returns whether or not deps successfully finished.\n \"\"\"\n while deps_timeout_s == RetryAction.WAIT_FOREVER or self._elapsed.seconds < deps_timeout_s:\n if await self.can_retry():\n return True\n wait_for = poll_interval_s\n if deps_timeout_s != RetryAction.WAIT_FOREVER:\n wait_for = min(wait_for, int(deps_timeout_s - self._elapsed.seconds))\n await asyncio.sleep(wait_for)\n self._elapsed += datetime.timedelta(seconds=wait_for)\n\n return await self.can_retry()\n\n async def issue_retry(self) -> bool:\n self._log(\"Issuing retry request\")\n response = await asyncio.get_event_loop().run_in_executor(\n None,\n functools.partial(\n client.request,\n urljoin(self.tron_client.url_base, self.action_run_id.url),\n data=self.retry_params,\n user_attribution=True,\n ),\n )\n if response.error:\n self._log(f\"Error: couldn't issue retry request: {response.content}\")\n self._retry_request_result = RetryAction.RETRY_FAIL\n else:\n self._log(f\"Got result: {response.content.get('result')}\")\n self._log(f\"Check the status of the retry run using: `tronview {self.full_action_name}`\")\n self._retry_request_result = RetryAction.RETRY_SUCCESS\n return self._retry_request_result\n\n\ndef retry_actions(\n tron_server: str,\n full_action_names: list[str],\n use_latest_command: bool = False,\n deps_timeout_s: int = RetryAction.NO_TIMEOUT,\n) -> list[RetryAction]:\n tron_client = client.Client(tron_server, user_attribution=True)\n r_actions = [RetryAction(tron_client, name, use_latest_command=use_latest_command) for name in full_action_names]\n\n loop = asyncio.get_event_loop()\n try:\n # first action starts checking immediately, rest have a jitter\n loop.run_until_complete(\n asyncio.gather(\n r_actions[0].wait_and_retry(deps_timeout_s=deps_timeout_s, jitter=False),\n *[ra.wait_and_retry(deps_timeout_s=deps_timeout_s) for ra in r_actions[1:]],\n )\n )\n finally:\n loop.close()\n return r_actions\n\n\nclass DisplayRetries(display.TableDisplay):\n\n columns = [\"Action Name\", \"Final Status\"]\n fields = [\"full_action_name\", \"status\"]\n widths = [60, 60]\n title = \"Retries\"\n resize_fields = {\"full_action_name\", \"status\"}\n header_color = \"hgray\"\n\n\ndef print_retries_table(retries: list[RetryAction]) -> None:\n \"\"\"Prints retry runs in a table\"\"\"\n with display.Color.enable():\n table = DisplayRetries().format([dict(full_action_name=r.full_action_name, status=r.status) for r in retries])\n print(table)\n" }, { "path": "tron/config/__init__.py", "content": "class ConfigError(Exception):\n \"\"\"Generic exception class for errors with config validation\"\"\"\n\n pass\n" }, { "path": "tron/config/config_parse.py", "content": "\"\"\"\nParse a dictionary structure and return an immutable structure that\ncontain a validated configuration.\n\n\nWARNING: it is *NOT* safe to delete classes that are being validated (or their attributes) if there are any references to them in DynamoDB until TRON-2200 is complete! (See DAR-2328)\nNOTE: this means that reverting a change that adds a new attribute is not safe :)\n\"\"\"\nimport datetime\nimport getpass\nimport itertools\nimport logging\nimport os\nfrom copy import deepcopy\nfrom typing import Any\nfrom urllib.parse import urlparse\n\nimport pytz\nfrom task_processing.plugins.mesos.constraints import OPERATORS\n\nfrom tron import command_context\nfrom tron.config import config_utils\nfrom tron.config import ConfigError\nfrom tron.config import schema\nfrom tron.config.config_utils import build_dict_name_validator\nfrom tron.config.config_utils import build_dict_value_validator\nfrom tron.config.config_utils import build_list_of_type_validator\nfrom tron.config.config_utils import ConfigContext\nfrom tron.config.config_utils import PartialConfigContext\nfrom tron.config.config_utils import StringFormatter\nfrom tron.config.config_utils import valid_bool\nfrom tron.config.config_utils import valid_dict\nfrom tron.config.config_utils import valid_exit_code\nfrom tron.config.config_utils import valid_float\nfrom tron.config.config_utils import valid_identifier\nfrom tron.config.config_utils import valid_int\nfrom tron.config.config_utils import valid_list\nfrom tron.config.config_utils import valid_name_identifier\nfrom tron.config.config_utils import valid_string\nfrom tron.config.config_utils import Validator\nfrom tron.config.schedule_parse import valid_schedule\nfrom tron.config.schema import CLEANUP_ACTION_NAME\nfrom tron.config.schema import ConfigAction\nfrom tron.config.schema import ConfigCleanupAction\nfrom tron.config.schema import ConfigConstraint\nfrom tron.config.schema import ConfigFieldSelectorSource\nfrom tron.config.schema import ConfigJob\nfrom tron.config.schema import ConfigKubernetes\nfrom tron.config.schema import ConfigMesos\nfrom tron.config.schema import ConfigNodeAffinity\nfrom tron.config.schema import ConfigParameter\nfrom tron.config.schema import ConfigProjectedSAVolume\nfrom tron.config.schema import ConfigSecretSource\nfrom tron.config.schema import ConfigSecretVolume\nfrom tron.config.schema import ConfigSecretVolumeItem\nfrom tron.config.schema import ConfigSSHOptions\nfrom tron.config.schema import ConfigState\nfrom tron.config.schema import ConfigTopologySpreadConstraints\nfrom tron.config.schema import ConfigVolume\nfrom tron.config.schema import MASTER_NAMESPACE\nfrom tron.config.schema import NamedTronConfig\nfrom tron.config.schema import TronConfig\n\nlog = logging.getLogger(__name__)\n\n\ndef build_format_string_validator(context_object):\n \"\"\"Validate that a string does not contain any unexpected formatting keys.\n valid_keys - a sequence of strings\n \"\"\"\n\n def validator(value, config_context):\n if config_context.partial:\n return valid_string(value, config_context)\n\n context = command_context.CommandContext(\n context_object,\n config_context.command_context,\n )\n\n try:\n StringFormatter(context).format(value)\n return value\n except (KeyError, ValueError) as e:\n error_msg = \"Unknown context variable %s at %s: %s\"\n raise ConfigError(error_msg % (e, config_context.path, value))\n except TypeError as e:\n error_msg = \"Wrong command format %s: %s at %s\"\n raise ConfigError(error_msg % (value, e, config_context.path))\n\n return validator\n\n\ndef valid_output_stream_dir(output_dir, config_context):\n \"\"\"Returns a valid string for the output directory, or raises ConfigError\n if the output_dir is not valid.\n \"\"\"\n if not output_dir:\n return\n\n if config_context.partial:\n return output_dir\n\n valid_string(output_dir, config_context)\n if not os.path.isdir(output_dir):\n msg = \"output_stream_dir '%s' is not a directory\"\n raise ConfigError(msg % output_dir)\n\n if not os.access(output_dir, os.W_OK):\n raise ConfigError(\n \"output_stream_dir '%s' is not writable\" % output_dir,\n )\n\n return output_dir\n\n\ndef valid_identity_file(file_path, config_context):\n valid_string(file_path, config_context)\n\n if config_context.partial:\n return file_path\n\n file_path = os.path.expanduser(file_path)\n if not os.path.exists(file_path):\n raise ConfigError(\"Private key file %s doesn't exist\" % file_path)\n\n public_key_path = file_path + \".pub\"\n if not os.path.exists(public_key_path):\n raise ConfigError(\"Public key file %s doesn't exist\" % public_key_path)\n return file_path\n\n\ndef valid_known_hosts_file(file_path, config_context):\n valid_string(file_path, config_context)\n\n if config_context.partial:\n return file_path\n\n file_path = os.path.expanduser(file_path)\n if not os.path.exists(file_path):\n raise ConfigError(\"Known hosts file %s doesn't exist\" % file_path)\n return file_path\n\n\ndef valid_command_context(context, config_context):\n # context can be any dict.\n return valid_dict(context or {}, config_context)\n\n\ndef valid_time_zone(tz, config_context):\n if tz is None:\n return None\n valid_string(tz, config_context)\n try:\n return pytz.timezone(tz)\n except pytz.exceptions.UnknownTimeZoneError:\n raise ConfigError(\"%s is not a valid time zone\" % tz)\n\n\ndef valid_node_name(value, config_context):\n valid_identifier(value, config_context)\n if not config_context.partial and value not in config_context.nodes:\n msg = \"Unknown node name %s at %s\"\n raise ConfigError(msg % (value, config_context.path))\n return value\n\n\ndef valid_master_address(value, config_context):\n \"\"\"Validates and normalizes Mesos master address.\n\n Must be HTTP or not include a scheme, and only include\n a host, without any path components.\n \"\"\"\n valid_string(value, config_context)\n\n # Parse with HTTP as default, only HTTP allowed.\n scheme, netloc, path, params, query, fragment = urlparse(value, \"http\")\n if scheme != \"http\":\n msg = f\"Only HTTP supported for Mesos master address, got {value}\"\n raise ConfigError(msg)\n\n if params or query or fragment:\n msg = f\"Mesos master address may not contain path components, got {value}\"\n raise ConfigError(msg)\n\n # Only one of netloc or path allowed, and no / except trailing ones.\n # netloc is empty if there's no scheme, then we try the path.\n path = path.rstrip(\"/\")\n if (netloc and path) or \"/\" in path:\n msg = f\"Mesos master address may not contain path components, got {value}\"\n raise ConfigError(msg)\n\n if not netloc:\n netloc = path\n\n if not netloc:\n msg = f\"Mesos master address is missing host, got {value}\"\n raise ConfigError(msg)\n\n return f\"{scheme}://{netloc}\"\n\n\ndef valid_k8s_master_address(value: str, config_context: ConfigContext) -> str:\n \"\"\"Validates and normalizes Kubernetes master address.\n\n Must be HTTP or not include a scheme, and only include\n a host, without any path components.\n \"\"\"\n valid_string(value, config_context)\n\n # Parse with HTTP as default, only HTTPS allowed.\n scheme, netloc, path, params, query, fragment = urlparse(url=value, scheme=\"https\")\n if scheme != \"http\":\n msg = f\"Only HTTPS supported for Kubernetes master address, got {value}\"\n raise ConfigError(msg)\n\n if params or query or fragment:\n msg = f\"Kubernetes master address may not contain path components, got {value}\"\n raise ConfigError(msg)\n\n # Only one of netloc or path allowed, and no / except trailing ones.\n path = path.rstrip(\"/\")\n if (netloc and path) or \"/\" in path:\n msg = f\"Kubernetes master address may not contain path components, got {value}\"\n raise ConfigError(msg)\n\n # netloc is empty if there's no scheme, so we fallback to path.\n if not netloc and path:\n netloc = path\n\n if not netloc:\n msg = f\"Kubernetes master address is missing host, got {value}\"\n raise ConfigError(msg)\n\n return f\"{scheme}://{netloc}\"\n\n\nclass ValidateConstraint(Validator):\n config_class = ConfigConstraint\n validators = {\n \"attribute\": valid_string,\n \"operator\": config_utils.build_enum_validator(OPERATORS.keys()),\n \"value\": valid_string,\n }\n\n\nvalid_constraint = ValidateConstraint()\n\n\nclass ValidateDockerParameter(Validator):\n config_class = ConfigParameter\n validators = {\n \"key\": valid_string,\n \"value\": valid_string,\n }\n\n\nvalid_docker_parameter = ValidateDockerParameter()\n\n\nclass ValidateVolume(Validator):\n config_class = ConfigVolume\n validators = {\n \"container_path\": valid_string,\n \"host_path\": valid_string,\n \"mode\": config_utils.build_real_enum_validator(schema.VolumeModes),\n }\n\n\nvalid_volume = ValidateVolume()\n\n\nclass ValidateSecretSource(Validator):\n config_class = ConfigSecretSource\n validators = {\n \"secret_name\": valid_string, # name of Kubernetes Secret\n \"key\": valid_string, # key name in Secret data\n }\n\n\nvalid_secret_source = ValidateSecretSource()\n\n\ndef valid_permission_mode(value: str | int, config_context: ConfigContext) -> str:\n try:\n decimal_value = int(\n str(value), base=8\n ) # take in permission mode as string or int representation of an octal number. Goes from 0 to 4095 in decimal.\n except ValueError:\n error_msg = \"Could not parse {} as octal permission mode at {}\"\n raise ConfigError(error_msg.format(value, config_context.path))\n if decimal_value > 4095 or decimal_value < 0:\n error_msg = \"Octal permission mode {} out of bound at {}\"\n raise ConfigError(error_msg.format(value, config_context.path))\n return str(value)\n\n\nclass ValidateSecretVolumeItem(Validator):\n config_class = ConfigSecretVolumeItem\n\n validators = {\n \"key\": valid_string, # name of current secret\n \"path\": valid_string, # New secret filename\n \"mode\": valid_permission_mode, # Octal permission mode\n }\n\n\nvalid_secret_volume_item = ValidateSecretVolumeItem()\n\n\nclass ValidateSecretVolume(Validator):\n config_class = ConfigSecretVolume\n\n optional = True\n defaults = {\n \"default_mode\": \"0644\",\n \"items\": None,\n }\n\n validators = {\n \"container_path\": valid_string,\n \"secret_volume_name\": valid_string,\n \"secret_name\": valid_string,\n \"default_mode\": valid_permission_mode,\n \"items\": build_list_of_type_validator(valid_secret_volume_item, allow_empty=True),\n }\n\n def post_validation(self, valid_input, config_context):\n \"\"\"Propagate default mode and enforce the secret-key match.\"\"\"\n # Ensure 'items' is an iterable list, even if defaulted to None by set_defaults.\n # The 'or []' handles the case where valid_input.get('items') returns None.\n items = valid_input.get(\"items\") or []\n\n # Our secrets will really only ever have one key, so weirdly we only care about a single\n # item of this array AND it must have the same name as the secret (which is the single key).\n if len(items) > 1:\n raise ConfigError(\n \"There is more than one item in the items array. This is unsupported as we don't support multi-key secrets.\"\n )\n\n processed_items = []\n modified = False\n default_mode = valid_input.get(\"default_mode\", self.defaults[\"default_mode\"])\n secret_name = valid_input.get(\"secret_name\")\n\n for item in items:\n if item.key != secret_name:\n raise ConfigError(f\"Item key '{item.key}' does not match the volume's secret name '{secret_name}'\")\n\n final_item = item\n if item.mode is None:\n # Apply volume's default_mode to items without an explicit mode.\n final_item = item._replace(mode=default_mode)\n modified = True\n processed_items.append(final_item)\n\n if modified:\n # Update valid_input with the (potentially) modified items tuple.\n # This ensures the final object reflects applied defaults.\n valid_input[\"items\"] = tuple(processed_items)\n\n\nvalid_secret_volume = ValidateSecretVolume()\n\n\nclass ValidateProjectedSAVolume(Validator):\n config_class = ConfigProjectedSAVolume\n optional = True\n defaults = {\n \"expiration_seconds\": 1800,\n }\n validators = {\n \"container_path\": valid_string,\n \"audience\": valid_string,\n \"expiration_seconds\": valid_int,\n }\n\n\nvalid_projected_sa_volume = ValidateProjectedSAVolume()\n\n\nclass ValidateFieldSelectorSource(Validator):\n config_class = ConfigFieldSelectorSource\n validators = {\n \"field_path\": valid_string, # k8s field path - e.g., `status.podIP`\n }\n\n\nvalid_field_selector_source = ValidateFieldSelectorSource()\n\n\ndef _valid_node_affinity_operator(value: str, config_context: ConfigContext) -> str:\n valid_operators = {\"In\", \"NotIn\", \"Exists\", \"NotExists\", \"Gt\", \"Lt\"}\n if value not in valid_operators:\n raise ConfigError(f\"Got {value} as a node affinity operator, expected one of {valid_operators}\")\n\n return value\n\n\nclass ValidateNodeAffinity(Validator):\n config_class = ConfigNodeAffinity\n validators = {\n \"key\": valid_string,\n \"operator\": _valid_node_affinity_operator,\n \"value\": build_list_of_type_validator(valid_string, allow_empty=True),\n }\n\n\nvalid_node_affinity = ValidateNodeAffinity()\n\n\ndef _valid_when_unsatisfiable(value: str, config_context: ConfigContext) -> str:\n valid_values = {\"DoNotSchedule\", \"ScheduleAnyway\"}\n if value not in valid_values:\n raise ConfigError(f\"Got {value} as a when_unsatisfiable value, expected one of {valid_values}\")\n\n return value\n\n\ndef _valid_topology_spread_label_selector(value: dict[str, str], config_context: ConfigContext) -> dict[str, str]:\n if not value:\n raise ConfigError(\"TopologySpreadConstraints must have a label_selector\")\n\n # XXX: we probably also want to enforce k8s limits for label lengths and whatnot\n if not all(isinstance(k, str) for k in value.keys()):\n raise ConfigError(\"TopologySpreadConstraints label_selector keys must be strings\")\n\n if not all(isinstance(s, str) for s in value.values()):\n raise ConfigError(\"TopologySpreadConstraints label_selector values must be strings\")\n\n return value\n\n\nclass ValidateTopologySpreadConstraints(Validator):\n config_class = ConfigTopologySpreadConstraints\n validators = {\n \"max_skew\": valid_int,\n \"when_unsatisfiable\": _valid_when_unsatisfiable,\n \"topology_key\": valid_string,\n \"label_selector\": _valid_topology_spread_label_selector,\n }\n\n\nvalid_topology_spread_constraints = ValidateTopologySpreadConstraints()\n\n\nclass ValidateSSHOptions(Validator):\n \"\"\"Validate SSH options.\"\"\"\n\n config_class = ConfigSSHOptions\n optional = True\n defaults = {\n \"agent\": False,\n \"identities\": (),\n \"known_hosts_file\": None,\n \"connect_timeout\": 30,\n \"idle_connection_timeout\": 3600,\n \"jitter_min_load\": 4,\n \"jitter_max_delay\": 20,\n \"jitter_load_factor\": 1,\n }\n\n validators = {\n \"agent\": valid_bool,\n # TODO: move this config and validations outside master namespace\n # 'identities': build_list_of_type_validator(\n # valid_identity_file, allow_empty=True),\n \"identities\": build_list_of_type_validator(\n valid_string,\n allow_empty=True,\n ),\n # 'known_hosts_file': valid_known_hosts_file,\n \"known_hosts_file\": valid_string,\n \"connect_timeout\": config_utils.valid_int,\n \"idle_connection_timeout\": config_utils.valid_int,\n \"jitter_min_load\": config_utils.valid_int,\n \"jitter_max_delay\": config_utils.valid_int,\n \"jitter_load_factor\": config_utils.valid_int,\n }\n\n def post_validation(self, valid_input, config_context):\n if config_context.partial:\n return\n\n if valid_input[\"agent\"] and \"SSH_AUTH_SOCK\" not in os.environ:\n raise ConfigError(\"No SSH Agent available ($SSH_AUTH_SOCK)\")\n\n\nvalid_ssh_options = ValidateSSHOptions()\n\n\nclass ValidateNode(Validator):\n config_class = schema.ConfigNode\n validators = {\n \"name\": config_utils.valid_identifier,\n \"username\": config_utils.valid_string,\n \"hostname\": config_utils.valid_string,\n \"port\": config_utils.valid_int,\n }\n\n defaults = {\n \"port\": 22,\n \"username\": getpass.getuser(),\n }\n\n def do_shortcut(self, node):\n \"\"\"Nodes can be specified with just a hostname string.\"\"\"\n if isinstance(node, str):\n return schema.ConfigNode(hostname=node, name=node, **self.defaults)\n\n def set_defaults(self, output_dict, config_context):\n super().set_defaults(output_dict, config_context)\n output_dict.setdefault(\"name\", output_dict[\"hostname\"])\n\n\nvalid_node = ValidateNode()\n\n\nclass ValidateNodePool(Validator):\n config_class = schema.ConfigNodePool\n validators = {\n \"name\": valid_identifier,\n \"nodes\": build_list_of_type_validator(valid_identifier),\n }\n\n def cast(self, node_pool, _context):\n if isinstance(node_pool, list):\n node_pool = dict(nodes=node_pool)\n return node_pool\n\n def set_defaults(self, node_pool, _):\n node_pool.setdefault(\"name\", \"_\".join(node_pool[\"nodes\"]))\n\n\nvalid_node_pool = ValidateNodePool()\n\n\ndef valid_action_name(value, config_context):\n valid_identifier(value, config_context)\n if value == CLEANUP_ACTION_NAME:\n error_msg = \"Invalid action name %s at %s\"\n raise ConfigError(error_msg % (value, config_context.path))\n return value\n\n\naction_context = command_context.build_filled_context(\n command_context.JobContext,\n command_context.JobRunContext,\n command_context.ActionRunContext,\n)\n\n\ndef valid_mesos_action(action, config_context):\n required_keys = {\"cpus\", \"mem\", \"docker_image\"}\n if action.get(\"executor\") == schema.ExecutorTypes.mesos.value:\n missing_keys = required_keys - set(action.keys())\n if missing_keys:\n raise ConfigError(\n \"Mesos executor for action {id} is missing these required keys: {keys}\".format(\n id=action[\"name\"],\n keys=missing_keys,\n ),\n )\n\n\ndef valid_kubernetes_action(action, config_context):\n required_keys = {\"cpus\", \"mem\", \"docker_image\"}\n if action.get(\"executor\") == schema.ExecutorTypes.kubernetes.value:\n missing_keys = required_keys - set(action.keys())\n if missing_keys:\n raise ConfigError(\n \"Kubernetes executor for action {id} is missing these required keys: {keys}\".format(\n id=action[\"name\"],\n keys=missing_keys,\n ),\n )\n\n\ndef valid_trigger_downstreams(trigger_downstreams, config_context):\n if isinstance(trigger_downstreams, (type(None), bool, dict)):\n return trigger_downstreams\n raise ConfigError(\"must be None, bool or dict\")\n\n\nclass ValidateAction(Validator):\n \"\"\"Validate an action.\"\"\"\n\n config_class = ConfigAction\n\n defaults = {\n \"node\": None,\n \"requires\": (),\n \"retries\": None,\n \"retries_delay\": None,\n \"expected_runtime\": datetime.timedelta(hours=24),\n \"executor\": schema.ExecutorTypes.ssh.value,\n \"cpus\": None,\n \"mem\": None,\n \"disk\": None,\n \"cap_add\": None,\n \"cap_drop\": None,\n \"constraints\": None,\n \"docker_image\": None,\n \"docker_parameters\": None,\n \"env\": None,\n \"secret_env\": None,\n \"secret_volumes\": None,\n \"projected_sa_volumes\": None,\n \"field_selector_env\": None,\n \"extra_volumes\": None,\n \"trigger_downstreams\": None,\n \"triggered_by\": None,\n \"on_upstream_rerun\": None,\n \"trigger_timeout\": None,\n \"node_selectors\": None,\n \"node_affinities\": None,\n \"topology_spread_constraints\": None,\n \"labels\": None,\n \"annotations\": None,\n \"service_account_name\": None,\n \"ports\": None,\n }\n requires = build_list_of_type_validator(\n valid_action_name,\n allow_empty=True,\n )\n validators = {\n \"name\": valid_action_name,\n \"command\": build_format_string_validator(action_context),\n \"node\": valid_node_name,\n \"requires\": requires,\n \"retries\": valid_int,\n \"retries_delay\": config_utils.valid_time_delta,\n \"expected_runtime\": config_utils.valid_time_delta,\n \"executor\": config_utils.build_real_enum_validator(schema.ExecutorTypes),\n \"cpus\": valid_float,\n \"mem\": valid_float,\n \"disk\": valid_float,\n \"cap_add\": valid_list,\n \"cap_drop\": valid_list,\n \"constraints\": build_list_of_type_validator(valid_constraint, allow_empty=True),\n \"docker_image\": valid_string,\n \"docker_parameters\": build_list_of_type_validator(\n valid_docker_parameter,\n allow_empty=True,\n ),\n \"env\": valid_dict,\n \"secret_env\": build_dict_value_validator(valid_secret_source),\n \"secret_volumes\": build_list_of_type_validator(valid_secret_volume, allow_empty=True),\n \"projected_sa_volumes\": build_list_of_type_validator(valid_projected_sa_volume, allow_empty=True),\n \"field_selector_env\": build_dict_value_validator(valid_field_selector_source),\n \"extra_volumes\": build_list_of_type_validator(valid_volume, allow_empty=True),\n \"trigger_downstreams\": valid_trigger_downstreams,\n \"triggered_by\": build_list_of_type_validator(valid_string, allow_empty=True),\n \"on_upstream_rerun\": config_utils.build_real_enum_validator(schema.ActionOnRerun),\n \"trigger_timeout\": config_utils.valid_time_delta,\n \"node_selectors:\": valid_dict,\n \"node_affinities\": build_list_of_type_validator(valid_node_affinity, allow_empty=True),\n \"topology_spread_constraints\": build_list_of_type_validator(\n valid_topology_spread_constraints, allow_empty=True\n ),\n \"labels:\": valid_dict,\n \"annotations\": valid_dict,\n \"service_account_name\": valid_string,\n \"ports\": build_list_of_type_validator(valid_int, allow_empty=True),\n }\n\n def post_validation(self, action, config_context):\n valid_mesos_action(action, config_context)\n valid_kubernetes_action(action, config_context)\n\n\nvalid_action = ValidateAction()\n\n\ndef valid_cleanup_action_name(value, config_context):\n if value != CLEANUP_ACTION_NAME:\n msg = \"Cleanup actions cannot have custom names %s.%s\"\n raise ConfigError(msg % (config_context.path, value))\n return CLEANUP_ACTION_NAME\n\n\nclass ValidateCleanupAction(Validator):\n config_class = ConfigCleanupAction\n defaults = {\n \"node\": None,\n \"name\": CLEANUP_ACTION_NAME,\n \"retries\": None,\n \"retries_delay\": None,\n \"expected_runtime\": datetime.timedelta(hours=24),\n \"executor\": schema.ExecutorTypes.ssh.value,\n \"cpus\": None,\n \"mem\": None,\n \"disk\": None,\n \"cap_add\": None,\n \"cap_drop\": None,\n \"constraints\": None,\n \"docker_image\": None,\n \"docker_parameters\": None,\n \"env\": None,\n \"secret_env\": None,\n \"secret_volumes\": None,\n \"projected_sa_volumes\": None,\n \"field_selector_env\": None,\n \"extra_volumes\": None,\n \"trigger_downstreams\": None,\n \"triggered_by\": None,\n \"on_upstream_rerun\": None,\n \"trigger_timeout\": None,\n \"node_selectors\": None,\n \"node_affinities\": None,\n \"topology_spread_constraints\": None,\n \"labels\": None,\n \"annotations\": None,\n \"service_account_name\": None,\n \"ports\": None,\n }\n validators = {\n \"name\": valid_cleanup_action_name,\n \"command\": build_format_string_validator(action_context),\n \"node\": valid_node_name,\n \"retries\": valid_int,\n \"retries_delay\": config_utils.valid_time_delta,\n \"expected_runtime\": config_utils.valid_time_delta,\n \"executor\": config_utils.build_real_enum_validator(schema.ExecutorTypes),\n \"cpus\": valid_float,\n \"mem\": valid_float,\n \"disk\": valid_float,\n \"cap_add\": valid_list,\n \"cap_drop\": valid_list,\n \"constraints\": build_list_of_type_validator(valid_constraint, allow_empty=True),\n \"docker_image\": valid_string,\n \"docker_parameters\": build_list_of_type_validator(\n valid_docker_parameter,\n allow_empty=True,\n ),\n \"env\": valid_dict,\n \"secret_env\": build_dict_value_validator(valid_secret_source),\n \"secret_volumes\": build_list_of_type_validator(valid_secret_volume, allow_empty=True),\n \"projected_sa_volumes\": build_list_of_type_validator(valid_projected_sa_volume, allow_empty=True),\n \"field_selector_env\": build_dict_value_validator(valid_field_selector_source),\n \"extra_volumes\": build_list_of_type_validator(valid_volume, allow_empty=True),\n \"trigger_downstreams\": valid_trigger_downstreams,\n \"triggered_by\": build_list_of_type_validator(valid_string, allow_empty=True),\n \"on_upstream_rerun\": config_utils.build_real_enum_validator(schema.ActionOnRerun),\n \"trigger_timeout\": config_utils.valid_time_delta,\n \"node_selectors:\": valid_dict,\n \"node_affinities\": build_list_of_type_validator(valid_node_affinity, allow_empty=True),\n \"topology_spread_constraints\": build_list_of_type_validator(\n valid_topology_spread_constraints, allow_empty=True\n ),\n \"labels\": valid_dict,\n \"annotations\": valid_dict,\n \"service_account_name\": valid_string,\n \"ports\": build_list_of_type_validator(valid_int, allow_empty=True),\n }\n\n def post_validation(self, action, config_context):\n valid_mesos_action(action, config_context)\n valid_kubernetes_action(action, config_context)\n\n\nvalid_cleanup_action = ValidateCleanupAction()\n\n\nclass ValidateJob(Validator):\n \"\"\"Validate jobs.\"\"\"\n\n config_class = ConfigJob\n defaults: dict[str, Any] = {\n \"run_limit\": 50,\n \"all_nodes\": False,\n \"cleanup_action\": None,\n \"enabled\": True,\n \"queueing\": True,\n \"allow_overlap\": False,\n \"max_runtime\": None,\n \"monitoring\": {},\n \"time_zone\": None,\n \"expected_runtime\": datetime.timedelta(hours=24),\n \"use_k8s\": False,\n }\n\n validators = {\n \"name\": valid_name_identifier,\n \"schedule\": valid_schedule,\n \"run_limit\": valid_int,\n \"all_nodes\": valid_bool,\n \"actions\": build_dict_name_validator(valid_action),\n \"cleanup_action\": valid_cleanup_action,\n \"node\": valid_node_name,\n \"queueing\": valid_bool,\n \"enabled\": valid_bool,\n \"allow_overlap\": valid_bool,\n \"max_runtime\": config_utils.valid_time_delta,\n \"monitoring\": valid_dict,\n \"time_zone\": valid_time_zone,\n \"expected_runtime\": config_utils.valid_time_delta,\n \"use_k8s\": valid_bool,\n }\n\n def cast(self, in_dict, config_context):\n in_dict[\"namespace\"] = config_context.namespace\n return in_dict\n\n # TODO: extract common code to a util function\n def _validate_dependencies(\n self,\n job: dict[str, Any], # TODO: create TypedDict for this\n # TODO: setup UniqueNameDict for use with mypy so that the following line\n # is not a lie\n actions: dict[str, ConfigAction],\n base_action: ConfigAction,\n current_action: ConfigAction | None = None,\n stack: list[str] | None = None,\n already_validated: set[tuple[str, str]] | None = None,\n ) -> None:\n \"\"\"Check for circular or misspelled dependencies.\"\"\"\n # for large graphs, we can end up validating the same jobs/actions repeatedly\n # this is unnecessary and we can skip a ton of work simply by caching what we've\n # already validated\n already_validated = already_validated or set()\n current_action = current_action or base_action\n validated = (job[\"name\"], current_action.name)\n if validated in already_validated:\n return None\n else:\n already_validated.add(validated)\n\n stack = stack or []\n stack.append(current_action.name)\n for dep in current_action.requires:\n if dep == base_action.name and len(stack) > 0:\n msg = \"Circular dependency in job.%s: %s\"\n raise ConfigError(msg % (job[\"name\"], \" -> \".join(stack)))\n if dep not in actions:\n raise ConfigError(\n 'Action jobs.%s.%s has a dependency \"%s\"'\n \" that is not in the same job!\" % (job[\"name\"], current_action.name, dep),\n )\n self._validate_dependencies(job, actions, base_action, actions[dep], stack, already_validated)\n\n stack.pop()\n\n def post_validation(self, job, config_context):\n \"\"\"Validate actions for the job.\"\"\"\n for _, action in job[\"actions\"].items():\n self._validate_dependencies(job, job[\"actions\"], action)\n\n\nvalid_job = ValidateJob()\n\n\nclass ValidateActionRunner(Validator):\n config_class = schema.ConfigActionRunner\n optional = True\n defaults = {\n \"runner_type\": None,\n \"remote_exec_path\": \"\",\n \"remote_status_path\": \"/tmp\",\n }\n\n validators = {\n \"runner_type\": config_utils.build_real_enum_validator(schema.ActionRunnerTypes),\n \"remote_status_path\": valid_string,\n \"remote_exec_path\": valid_string,\n }\n\n\nclass ValidateStatePersistence(Validator):\n config_class = schema.ConfigState\n defaults = {\n \"buffer_size\": 1,\n \"dynamodb_region\": None,\n \"table_name\": None,\n \"max_transact_write_items\": 8,\n }\n\n validators = {\n \"name\": valid_string,\n \"store_type\": config_utils.build_real_enum_validator(schema.StatePersistenceTypes),\n \"buffer_size\": valid_int,\n \"dynamodb_region\": valid_string,\n \"table_name\": valid_string,\n \"max_transact_write_items\": valid_int,\n }\n\n def post_validation(self, config, config_context):\n buffer_size = config.get(\"buffer_size\")\n\n if buffer_size and buffer_size < 1:\n path = config_context.path\n raise ConfigError(\"%s buffer_size must be >= 1.\" % path)\n\n store_type = config.get(\"store_type\")\n\n if store_type == schema.StatePersistenceTypes.dynamodb.value:\n if not config.get(\"table_name\"):\n raise ConfigError(f\"{config_context.path} table_name is required when store_type is 'dynamodb'\")\n if not config.get(\"dynamodb_region\"):\n raise ConfigError(f\"{config_context.path} dynamodb_region is required when store_type is 'dynamodb'\")\n\n max_transact = config.get(\"max_transact_write_items\")\n\n # Upper bound is based on boto3 transact_write_items limit\n if not 1 <= max_transact <= 100:\n raise ConfigError(\n f\"{config_context.path} max_transact_write_items must be between 1 and 100, got {max_transact}\"\n )\n\n\nvalid_state_persistence = ValidateStatePersistence()\n\n\nclass ValidateMesos(Validator):\n config_class = ConfigMesos\n option = True\n defaults = {\n \"master_address\": None,\n \"master_port\": 5050,\n \"secret_file\": None,\n \"role\": \"*\",\n \"principal\": \"tron\",\n \"enabled\": False,\n \"default_volumes\": (),\n \"dockercfg_location\": None,\n \"offer_timeout\": 300,\n }\n\n validators = {\n \"master_address\": valid_master_address,\n \"master_port\": valid_int,\n \"secret\": valid_string,\n \"role\": valid_string,\n \"enabled\": valid_bool,\n \"default_volumes\": build_list_of_type_validator(valid_volume, allow_empty=True),\n \"dockercfg_location\": valid_string,\n \"offer_timeout\": valid_int,\n }\n\n\nvalid_mesos_options = ValidateMesos()\n\n\nclass ValidateKubernetes(Validator):\n config_class = ConfigKubernetes\n optional = True\n defaults = {\n \"kubeconfig_path\": None,\n \"enabled\": False,\n \"non_retryable_exit_codes\": (),\n \"default_volumes\": (),\n }\n\n validators = {\n \"kubeconfig_path\": valid_string,\n \"enabled\": valid_bool,\n \"non_retryable_exit_codes\": build_list_of_type_validator(valid_exit_code, allow_empty=True),\n \"default_volumes\": build_list_of_type_validator(valid_volume, allow_empty=True),\n \"watcher_kubeconfig_paths\": build_list_of_type_validator(valid_string, allow_empty=True),\n }\n\n\nvalid_kubernetes_options = ValidateKubernetes()\n\n\ndef validate_jobs(config, config_context):\n \"\"\"Validate jobs\"\"\"\n valid_jobs = build_dict_name_validator(valid_job, allow_empty=True)\n validation = [(\"jobs\", valid_jobs)]\n\n for config_name, valid in validation:\n child_context = config_context.build_child_context(config_name)\n config[config_name] = valid(config.get(config_name, []), child_context)\n\n fmt_string = \"Job names must be unique %s\"\n config_utils.unique_names(fmt_string, config[\"jobs\"])\n\n\nDEFAULT_STATE_PERSISTENCE = ConfigState(\n name=\"tron_state\",\n store_type=\"shelve\",\n buffer_size=1,\n)\nDEFAULT_NODE = ValidateNode().do_shortcut(node=\"localhost\")\n\n\nclass ValidateConfig(Validator):\n \"\"\"Given a parsed config file (should be only basic literals and\n containers), return an immutable, fully populated series of namedtuples and\n dicts with all defaults filled in, all valid values, and no unused\n values. Throws a ConfigError if any part of the input dict is invalid.\n \"\"\"\n\n config_class = TronConfig\n defaults = {\n \"action_runner\": {},\n \"output_stream_dir\": None,\n \"command_context\": {},\n \"ssh_options\": ConfigSSHOptions(**ValidateSSHOptions.defaults),\n \"time_zone\": None,\n \"state_persistence\": DEFAULT_STATE_PERSISTENCE,\n \"nodes\": {\n \"localhost\": DEFAULT_NODE,\n },\n \"node_pools\": {},\n \"jobs\": (),\n \"mesos_options\": ConfigMesos(**ValidateMesos.defaults),\n \"k8s_options\": ConfigKubernetes(**ValidateKubernetes.defaults),\n \"eventbus_enabled\": None,\n \"read_json\": False,\n }\n node_pools = build_dict_name_validator(valid_node_pool, allow_empty=True)\n nodes = build_dict_name_validator(valid_node, allow_empty=True)\n validators = {\n \"action_runner\": ValidateActionRunner(),\n \"output_stream_dir\": valid_output_stream_dir,\n \"command_context\": valid_command_context,\n \"ssh_options\": valid_ssh_options,\n \"time_zone\": valid_time_zone,\n \"state_persistence\": valid_state_persistence,\n \"nodes\": nodes,\n \"node_pools\": node_pools,\n \"mesos_options\": valid_mesos_options,\n \"k8s_options\": valid_kubernetes_options,\n \"eventbus_enabled\": valid_bool,\n \"read_json\": valid_bool,\n }\n optional = False\n\n def validate_node_pool_nodes(self, config):\n \"\"\"Validate that each node in a node_pool is in fact a node, and not\n another pool.\n \"\"\"\n all_node_names = set(config[\"nodes\"])\n for node_pool in config[\"node_pools\"].values():\n invalid_names = set(node_pool.nodes) - all_node_names\n if invalid_names:\n msg = \"NodePool %s contains other NodePools: \" % node_pool.name\n raise ConfigError(msg + \",\".join(invalid_names))\n\n def post_validation(self, config, _):\n \"\"\"Validate a non-named config.\"\"\"\n node_names = config_utils.unique_names(\n \"Node and NodePool names must be unique %s\",\n config[\"nodes\"],\n config.get(\"node_pools\", []),\n )\n\n if config.get(\"node_pools\"):\n self.validate_node_pool_nodes(config)\n\n config_context = ConfigContext(\n \"config\",\n node_names,\n config.get(\"command_context\"),\n MASTER_NAMESPACE,\n )\n validate_jobs(config, config_context)\n\n\nclass ValidateNamedConfig(Validator):\n \"\"\"A shorter validator for named configurations, which allow for\n jobs to be defined as configuration fragments that\n are, in turn, reconciled by Tron.\n \"\"\"\n\n config_class = NamedTronConfig\n type_name = \"NamedConfigFragment\"\n defaults = {\n \"jobs\": (),\n }\n\n optional = False\n\n def post_validation(self, config, config_context):\n validate_jobs(config, config_context)\n\n\nvalid_config = ValidateConfig()\nvalid_named_config = ValidateNamedConfig()\n\n\ndef validate_fragment(name, fragment, master_config=None):\n \"\"\"Validate a fragment with a partial context.\"\"\"\n config_context = PartialConfigContext(name, name)\n if name == MASTER_NAMESPACE:\n return valid_config(fragment, config_context=config_context)\n if master_config is None:\n return valid_named_config(fragment, config_context=config_context)\n\n config_mapping = {MASTER_NAMESPACE: master_config, name: fragment}\n for config_name, config in validate_config_mapping(config_mapping):\n if config_name == name:\n return config\n\n\ndef get_nodes_from_master_namespace(master):\n return set(itertools.chain(master.nodes, master.node_pools))\n\n\ndef validate_config_mapping(config_mapping):\n if MASTER_NAMESPACE not in config_mapping:\n msg = \"A config mapping requires a %s namespace\"\n raise ConfigError(msg % MASTER_NAMESPACE)\n\n # we mutate this mapping - so let's make sure that we're making a copy\n # in case the passed-in mapping is used elsewhere\n config_mapping_to_validate = deepcopy(config_mapping)\n master = valid_config(config_mapping_to_validate.pop(MASTER_NAMESPACE))\n nodes = get_nodes_from_master_namespace(master)\n yield MASTER_NAMESPACE, master\n\n for name, content in config_mapping_to_validate.items():\n context = ConfigContext(\n name,\n nodes,\n master.command_context,\n name,\n )\n yield name, valid_named_config(content, config_context=context)\n\n\nclass ConfigContainer:\n \"\"\"A container around configuration fragments (and master).\"\"\"\n\n def __init__(self, config_mapping):\n self.configs = config_mapping\n\n def items(self):\n return self.configs.items()\n\n @classmethod\n def create(cls, config_mapping):\n return cls(dict(validate_config_mapping(config_mapping)))\n\n # TODO: DRY with get_jobs()\n def get_job_names(self):\n job_names = []\n for config in self.configs.values():\n job_names.extend(config.jobs)\n return job_names\n\n def get_jobs(self):\n return dict(\n itertools.chain.from_iterable(config.jobs.items() for _, config in self.configs.items()),\n )\n\n def get_master(self):\n return self.configs[MASTER_NAMESPACE]\n\n def get_node_names(self):\n return get_nodes_from_master_namespace(self.get_master())\n\n def __getitem__(self, name):\n return self.configs[name]\n\n def __contains__(self, name):\n return name in self.configs\n" }, { "path": "tron/config/config_utils.py", "content": "\"\"\"Utilities used for configuration parsing and validation.\"\"\"\nimport datetime\nimport functools\nimport itertools\nimport re\nfrom string import Formatter\n\nfrom tron.config import ConfigError\nfrom tron.config.schema import MASTER_NAMESPACE\n\nMAX_IDENTIFIER_LENGTH = 255\nIDENTIFIER_RE = re.compile(r\"^[A-Za-z_][\\w\\-]{0,254}$\")\n\n\nclass StringFormatter(Formatter):\n def __init__(self, context=None):\n Formatter.__init__(self)\n self.context = context\n\n def get_value(self, key, args, kwds):\n if isinstance(key, str):\n try:\n return kwds[key]\n except KeyError:\n return self.context[key]\n else:\n return Formatter.get_value(key, args, kwds)\n\n\nclass UniqueNameDict(dict):\n \"\"\"A dict like object that throws a ConfigError if a key exists and\n __setitem__ is called to change the value of that key.\n\n fmt_string - format string used to create an error message, expects a\n single format argument of 'key'\n \"\"\"\n\n def __init__(self, fmt_string):\n super().__init__()\n self.fmt_string = fmt_string\n\n def __setitem__(self, key, value):\n if key in self:\n raise ConfigError(self.fmt_string % key)\n super().__setitem__(key, value)\n\n\ndef unique_names(fmt_string, *seqs):\n \"\"\"Validate that each object in all sequences has a unique name.\"\"\"\n name_dict = UniqueNameDict(fmt_string)\n for item in itertools.chain.from_iterable(seqs):\n name_dict[item] = True\n return name_dict\n\n\ndef build_type_validator(validator, error_fmt):\n \"\"\"Create a validator function using `validator` to validate the value.\n validator - a function which takes a single argument `value`\n error_fmt - a string which accepts two format variables (path, value)\n\n Returns a function func(value, config_context) where\n value - the value to validate\n config_context - a ConfigContext object\n Returns True if the value is valid\n \"\"\"\n\n def f(value, config_context):\n if not validator(value):\n raise ConfigError(error_fmt % (config_context.path, value))\n return value\n\n return f\n\n\ndef valid_number(type_func, value, config_context, allow_negative=False):\n path = config_context.path\n try:\n value = type_func(value)\n except TypeError:\n name = type_func.__name__\n raise ConfigError(f\"Value at {path} is not an {name}: {value}\")\n\n if value < 0 and not allow_negative:\n raise ConfigError(\"%s must be a positive int.\" % path)\n\n return value\n\n\nvalid_exit_code = functools.partial(valid_number, int, allow_negative=True)\nvalid_int = functools.partial(valid_number, int)\nvalid_float = functools.partial(valid_number, float)\n\nvalid_identifier = build_type_validator(\n lambda s: isinstance(s, str) and IDENTIFIER_RE.match(s),\n \"Identifier at %s is not a valid identifier: %s\",\n)\n\nvalid_list = build_type_validator(\n lambda s: isinstance(s, list),\n \"Value at %s is not a list: %s\",\n)\n\nvalid_string = build_type_validator(\n lambda s: isinstance(s, str),\n \"Value at %s is not a string: %s\",\n)\n\nvalid_dict = build_type_validator(\n lambda s: isinstance(s, dict),\n \"Value at %s is not a dictionary: %s\",\n)\n\nvalid_bool = build_type_validator(\n lambda s: isinstance(s, bool),\n \"Value at %s is not a boolean: %s\",\n)\n\n\ndef build_enum_validator(enum):\n enum = set(enum)\n msg = \"Value at %%s is not in %s: %%s.\" % str(enum)\n return build_type_validator(enum.__contains__, msg)\n\n\ndef build_real_enum_validator(enum):\n def enum_validator(value, config_context):\n try:\n return enum(value).value\n except Exception:\n raise ConfigError(\n f\"Value at {config_context.path} is not in {enum!r}: {value!r}\",\n )\n\n return enum_validator\n\n\ndef valid_time(value, config_context):\n valid_string(value, config_context)\n for format in [\"%H:%M\", \"%H:%M:%S\"]:\n try:\n return datetime.datetime.strptime(value, format)\n except ValueError:\n pass\n msg = \"Value at %s is not a valid time\"\n raise ConfigError(msg % config_context.path)\n\n\n# Translations from possible configuration units to the argument to\n# datetime.timedelta\nTIME_INTERVAL_MAPPING = {\n \"days\": [\"d\", \"day\", \"days\"],\n \"hours\": [\"h\", \"hr\", \"hrs\", \"hour\", \"hours\"],\n \"minutes\": [\"m\", \"min\", \"mins\", \"minute\", \"minutes\"],\n \"seconds\": [\"s\", \"sec\", \"secs\", \"second\", \"seconds\"],\n}\nTIME_INTERVAL_UNITS = {short: long for (long, short_list) in TIME_INTERVAL_MAPPING.items() for short in short_list}\n\nTIME_INTERVAL_RE = re.compile(r\"^\\s*(?P\\d+)\\s*(?P[a-zA-Z]+)\\s*$\")\n\n\ndef valid_time_delta(value, config_context):\n error_msg = \"Value at %s is not a valid time delta: %s\"\n matches = TIME_INTERVAL_RE.match(value)\n if not matches:\n raise ConfigError(error_msg % (config_context.path, value))\n\n units = matches.group(\"units\")\n if units not in TIME_INTERVAL_UNITS:\n raise ConfigError(error_msg % (config_context.path, value))\n\n time_spec = {TIME_INTERVAL_UNITS[units]: int(matches.group(\"value\"))}\n return datetime.timedelta(**time_spec)\n\n\ndef valid_name_identifier(value, config_context):\n valid_identifier(value, config_context)\n if config_context.partial:\n return value\n return f\"{config_context.namespace}.{value}\"\n\n\ndef build_list_of_type_validator(item_validator, allow_empty=False):\n \"\"\"Build a validator which validates a list contains items which pass\n item_validator.\n \"\"\"\n\n def validator(value, config_context):\n if allow_empty and not value:\n return ()\n seq = valid_list(value, config_context)\n if not seq:\n msg = \"Required non-empty list at %s\"\n raise ConfigError(msg % config_context.path)\n return tuple(item_validator(item, config_context) for item in seq)\n\n return validator\n\n\ndef build_dict_name_validator(item_validator, allow_empty=False):\n \"\"\"Build a validator which validates a list or dict, and returns a dict.\n Item validator must expect a \"name\" key, mapped to the key of the dict item\"\"\"\n valid = build_list_of_type_validator(item_validator, allow_empty)\n\n def validator(value, config_context):\n if isinstance(value, dict):\n value = [\n {\n \"name\": name,\n **config,\n }\n for name, config in value.items()\n ]\n\n msg = \"Duplicate name %%s at %s\" % config_context.path\n name_dict = UniqueNameDict(msg)\n for item in valid(value, config_context):\n name_dict[item.name] = item\n return name_dict\n\n return validator\n\n\ndef build_dict_value_validator(item_validator, allow_empty=False):\n \"\"\"Build a validator which validates values of a dict, and returns a dict\"\"\"\n\n def validator(value, config_context):\n if not isinstance(value, dict):\n msg = \"Require a dict of type %s at %s\"\n raise ConfigError(msg % (item_validator.type_name, config_context.path))\n result_dict = dict()\n for k, v in value.items():\n result_dict[k] = item_validator(v, config_context)\n return result_dict\n\n return validator\n\n\nclass ConfigContext:\n \"\"\"An object to encapsulate the context in a configuration file. Supplied\n to Validators to perform validation which requires knowledge of\n configuration outside of the immediate configuration dictionary.\n \"\"\"\n\n partial = False\n\n def __init__(self, path, nodes, command_context, namespace):\n self.path = path\n self.nodes = set(nodes or [])\n self.command_context = command_context or {}\n self.namespace = namespace\n\n def build_child_context(self, path):\n \"\"\"Construct a new ConfigContext based on this one.\"\"\"\n path = f\"{self.path}.{path}\"\n args = path, self.nodes, self.command_context, self.namespace\n return ConfigContext(*args)\n\n\nclass PartialConfigContext:\n \"\"\"A context object which has only a partial context. It is missing\n command_context and nodes. This is likely because it is being used in\n a named configuration fragment that does not have access to those pieces\n of the configuration.\n \"\"\"\n\n partial = True\n\n def __init__(self, path, namespace):\n self.path = path\n self.namespace = namespace\n\n def build_child_context(self, path):\n path = f\"{self.path}.{path}\"\n return PartialConfigContext(path, self.namespace)\n\n\nclass NullConfigContext:\n path = \"\"\n nodes = set() # type: ignore\n command_context = {} # type: ignore\n namespace = MASTER_NAMESPACE\n partial = False\n\n @staticmethod\n def build_child_context(_):\n return NullConfigContext\n\n\n# TODO: extract code\nclass Validator:\n \"\"\"Base class for validating a collection and creating a mutable\n collection from the source.\n \"\"\"\n\n config_class: type | None = None\n defaults = {} # type: ignore\n validators = {} # type: ignore\n optional = False\n\n def validate(self, in_dict, config_context):\n if self.optional and in_dict is None:\n return None\n\n if in_dict is None:\n raise ConfigError(\"A %s is required.\" % self.type_name)\n\n shortcut_value = self.do_shortcut(in_dict)\n if shortcut_value:\n return shortcut_value\n\n config_context = self.build_context(in_dict, config_context)\n in_dict = self.cast(in_dict, config_context)\n self.validate_required_keys(in_dict)\n self.validate_extra_keys(in_dict)\n return self.build_config(in_dict, config_context)\n\n def __call__(self, in_dict, config_context=NullConfigContext):\n return self.validate(in_dict, config_context)\n\n @property\n def type_name(self):\n \"\"\"Return a string that represents the config_class being validated.\n This name is used for error messages, so we strip off the word\n Config so the name better matches what the user sees in the config.\n \"\"\"\n return self.config_class.__name__.replace(\"Config\", \"\")\n\n @property\n def all_keys(self):\n return self.config_class.required_keys + self.config_class.optional_keys\n\n def do_shortcut(self, in_dict):\n \"\"\"Override if your validator can skip most of the validation by\n checking this condition. If this returns a truthy value, the\n validation will end immediately and return that value.\n \"\"\"\n pass\n\n def cast(self, in_dict, _):\n \"\"\"If your validator accepts input in different formations, override\n this method to cast your input into a common format.\n \"\"\"\n return in_dict\n\n def build_context(self, in_dict, config_context):\n path = self.path_name(in_dict.get(\"name\"))\n return config_context.build_child_context(path)\n\n def validate_required_keys(self, in_dict):\n \"\"\"Check that all required keys are present.\"\"\"\n missing_keys = set(self.config_class.required_keys) - set(in_dict)\n if not missing_keys:\n return\n\n missing_key_str = \", \".join(missing_keys)\n if \"name\" in self.all_keys and \"name\" in in_dict:\n msg = \"%s %s is missing options: %s\"\n name = in_dict[\"name\"]\n raise ConfigError(msg % (self.type_name, name, missing_key_str))\n\n msg = \"Nameless %s is missing options: %s\"\n raise ConfigError(msg % (self.type_name, missing_key_str))\n\n def validate_extra_keys(self, in_dict):\n \"\"\"Check that no unexpected keys are present.\"\"\"\n extra_keys = set(in_dict) - set(self.all_keys)\n if not extra_keys:\n return\n\n msg = \"Unknown keys in %s %s: %s\"\n name = in_dict.get(\"name\", \"\")\n raise ConfigError(msg % (self.type_name, name, \", \".join(extra_keys)))\n\n def set_defaults(self, output_dict, _config_context):\n \"\"\"Set any default values for any optional values that were not\n specified.\n \"\"\"\n for key, value in self.defaults.items():\n output_dict.setdefault(key, value)\n\n def path_name(self, name=None):\n return f\"{self.type_name}.{name}\" if name else self.type_name\n\n def post_validation(self, valid_input, config_context):\n \"\"\"Hook to perform additional validation steps after key validation\n completes.\n \"\"\"\n pass\n\n def build_config(self, in_dict, config_context):\n \"\"\"Construct the configuration by validating the contents, setting\n defaults, and returning an instance of the config_class.\n \"\"\"\n output_dict = self.validate_contents(in_dict, config_context)\n self.set_defaults(output_dict, config_context)\n self.post_validation(output_dict, config_context)\n\n return self.config_class(**output_dict)\n\n def validate_contents(self, input, config_context):\n \"\"\"Override this to validate each value in the input.\"\"\"\n valid_input = {}\n for key, value in input.items():\n if key in self.validators:\n child_context = config_context.build_child_context(key)\n valid_input[key] = self.validators[key](value, child_context)\n else:\n valid_input[key] = value\n return valid_input\n" }, { "path": "tron/config/manager.py", "content": "import hashlib\nimport logging\nimport os\nfrom copy import deepcopy\n\nfrom tron import yaml\nfrom tron.config import config_parse\nfrom tron.config import ConfigError\nfrom tron.config import schema\nfrom tron.core.jobgraph import JobGraph\nfrom tron.utils import maybe_decode\nfrom tron.utils import maybe_encode\n\nlog = logging.getLogger(__name__)\n\n\ndef from_string(content):\n try:\n return yaml.safe_load(content)\n except yaml.yaml.error.YAMLError as e:\n raise ConfigError(\"Invalid config format: %s\" % str(e))\n\n\ndef write(path, content):\n with open(path, \"w\") as fh:\n yaml.dump(content, fh)\n\n\ndef read(path):\n with open(path) as fh:\n return from_string(fh)\n\n\ndef write_raw(path, content):\n with open(path, \"w\") as fh:\n fh.write(\n maybe_decode(content)\n ) # TODO: TRON-2293 maybe_decode is a relic of Python2->Python3 migration. Remove it.\n\n\ndef read_raw(path: str) -> str:\n with open(path) as fh:\n return fh.read()\n\n\ndef hash_digest(content: str | bytes) -> str:\n return hashlib.sha1(\n maybe_encode(content)\n ).hexdigest() # TODO: TRON-2293 maybe_encode is a relic of Python2->Python3 migration. Remove it.\n\n\nclass ManifestFile:\n \"\"\"Manage the manifest file, which tracks name to filename.\"\"\"\n\n MANIFEST_FILENAME = \"_manifest.yaml\"\n\n def __init__(self, path):\n self.filename = os.path.join(path, self.MANIFEST_FILENAME)\n\n def create(self):\n if os.path.isfile(self.filename):\n msg = \"Refusing to create manifest. File %s exists.\"\n log.info(msg % self.filename)\n return\n\n write(self.filename, {})\n\n def add(self, name, filename):\n manifest = read(self.filename)\n manifest[name] = filename\n write(self.filename, manifest)\n\n def delete(self, name):\n manifest = read(self.filename)\n if name not in manifest:\n msg = \"Namespace %s does not exist in manifest, cannot delete.\"\n log.info(msg % name)\n return\n\n del manifest[name]\n write(self.filename, manifest)\n\n def get_file_mapping(self):\n return read(self.filename)\n\n def get_file_name(self, name):\n return self.get_file_mapping().get(name)\n\n def __contains__(self, name):\n return name in self.get_file_mapping()\n\n\nclass ConfigManager:\n \"\"\"Read, load and write configuration.\"\"\"\n\n DEFAULT_HASH = hash_digest(\"\")\n\n def __init__(self, config_path, manifest=None):\n self.config_path = config_path\n self.manifest = manifest or ManifestFile(config_path)\n self.name_mapping = None\n\n def build_file_path(self, name):\n name = name.replace(\".\", \"_\").replace(os.path.sep, \"_\")\n return os.path.join(self.config_path, \"%s.yaml\" % name)\n\n def read_raw_config(self, name: str = schema.MASTER_NAMESPACE) -> str:\n \"\"\"Read the config file without converting to yaml.\"\"\"\n filename = self.manifest.get_file_name(name)\n return read_raw(filename)\n\n def write_config(self, name: str, content: str) -> None:\n loaded_content = from_string(content)\n self.validate_with_fragment(\n name,\n content=loaded_content,\n # TODO: remove this constraint after tron triggers across clusters are supported.\n should_validate_missing_dependency=False,\n )\n # validate_with_fragment throws if the updated content is invalid - so if we get here\n # we know it's safe to reflect the update in our config store\n self.get_config_name_mapping()[name] = loaded_content\n\n # ...and then let's also persist the update to disk since memory is temporary, but disk is forever™\n filename = self.get_filename_from_manifest(name)\n write_raw(filename, content)\n\n def delete_config(self, name: str) -> None:\n filename = self.manifest.get_file_name(name)\n if not filename:\n msg = \"Namespace %s does not exist in manifest, cannot delete.\"\n log.info(msg % name)\n return\n\n # to avoid needing to reload from disk on every config load - we need to ensure that\n # we also persist config deletions into our cache\n self.get_config_name_mapping().pop(name, None)\n self.manifest.delete(name)\n os.remove(filename)\n\n def get_filename_from_manifest(self, name):\n def create_filename():\n filename = self.build_file_path(name)\n self.manifest.add(name, filename)\n return filename\n\n return self.manifest.get_file_name(name) or create_filename()\n\n def validate_with_fragment(\n self,\n name,\n content,\n should_validate_missing_dependency=True,\n ):\n # NOTE: we deepcopy rather than swap values to keep this a pure function\n # get_config_name_mapping() returns a shared dict, so this would otherwise\n # actually update the mapping - which would be unwanted/need to be rolled-back\n # should validation fail.\n name_mapping = deepcopy(self.get_config_name_mapping())\n name_mapping[name] = content\n try:\n JobGraph(\n config_parse.ConfigContainer.create(name_mapping),\n should_validate_missing_dependency=should_validate_missing_dependency,\n )\n except ValueError as e:\n raise ConfigError(str(e))\n\n def get_config_name_mapping(self):\n if self.name_mapping is None:\n log.info(\"Creating config mapping cache...\")\n seq = self.manifest.get_file_mapping().items()\n self.name_mapping = {name: read(filename) for name, filename in seq}\n return self.name_mapping\n\n def load(self):\n \"\"\"Return the fully constructed configuration.\"\"\"\n log.info(\"Loading full config from %s\" % self.config_path)\n name_mapping = self.get_config_name_mapping()\n return config_parse.ConfigContainer.create(name_mapping)\n\n def get_hash(self, name: str) -> str:\n \"\"\"Return a hash of the configuration contents for name.\"\"\"\n if name not in self:\n return self.DEFAULT_HASH\n\n if name in self.get_config_name_mapping():\n # unfortunately, we have the parsed dict in memory.\n # rather than hit the disk to get the raw string - let's convert\n # the in-memory dict to a yaml string and hash that to save a couple\n # ms (in testing, ~3ms over loading from disk and ~1ms over dumping to json :p)\n # TODO: consider storing the hash alongside the config so that we only calculate\n # hashes once?\n return hash_digest(\n yaml.dump(\n self.get_config_name_mapping()[name],\n # ensure that the keys are always in a stable order\n sort_keys=True,\n )\n )\n\n # the config for any name should always be in our name mapping\n # ...but just in case, let's fallback to reading from disk.\n log.warning(\"%s not found in name mapping - falling back to hashing contents on disk!\")\n return hash_digest(self.read_raw_config(name))\n\n def __contains__(self, name):\n return name in self.manifest\n\n def get_namespaces(self) -> list[str]:\n return list(self.manifest.get_file_mapping().keys())\n\n\ndef create_new_config(path, master_content):\n \"\"\"Create a new configuration directory with master config.\"\"\"\n os.makedirs(path)\n manager = ConfigManager(path)\n manager.manifest.create()\n filename = manager.get_filename_from_manifest(schema.MASTER_NAMESPACE)\n write_raw(filename, master_content)\n" }, { "path": "tron/config/schedule_parse.py", "content": "\"\"\"\nParse and validate scheduler configuration and return immutable structures.\n\"\"\"\nimport calendar\nimport datetime\nimport re\nfrom collections import namedtuple\n\nfrom tron.config import config_utils\nfrom tron.config import ConfigError\nfrom tron.config import schema\nfrom tron.utils import crontab\n\nConfigGenericSchedule = schema.config_object_factory(\n \"ConfigGenericSchedule\",\n [\"type\", \"value\"],\n [\"jitter\"],\n)\n\nConfigGrocScheduler = namedtuple(\n \"ConfigGrocScheduler\",\n \"original ordinals weekdays monthdays months timestr jitter\",\n)\n\nConfigCronScheduler = namedtuple(\n \"ConfigCronScheduler\",\n \"original minutes hours monthdays months weekdays ordinals jitter\",\n)\n\nConfigDailyScheduler = namedtuple(\n \"ConfigDailyScheduler\",\n \"original hour minute second days jitter\",\n)\n\n\nclass ScheduleParseError(ConfigError):\n pass\n\n\ndef pad_sequence(seq, size, padding=None):\n \"\"\"Force a sequence to size. Pad with padding if too short, and ignore\n extra pieces if too long.\"\"\"\n return (list(seq) + [padding for _ in range(size)])[:size]\n\n\ndef schedule_config_from_string(schedule, config_context):\n \"\"\"Return a scheduler config object from a string.\"\"\"\n schedule = schedule.strip()\n name, schedule_config = pad_sequence(\n schedule.split(None, 1),\n 2,\n padding=\"\",\n )\n if name not in schedulers:\n config = ConfigGenericSchedule(\"groc daily\", schedule, jitter=None)\n return parse_groc_expression(config, config_context)\n\n config = ConfigGenericSchedule(name, schedule_config, jitter=None)\n return validate_generic_schedule_config(config, config_context)\n\n\ndef validate_generic_schedule_config(config, config_context):\n return schedulers[config.type](config, config_context)\n\n\n# TODO: remove in 0.7\ndef schedule_config_from_legacy_dict(schedule, config_context):\n \"\"\"Support old style schedules as dicts.\"\"\"\n if \"start_time\" in schedule or \"days\" in schedule:\n start_time = schedule.get(\"start_time\", \"00:00:00\")\n days = schedule.get(\"days\", \"\")\n scheduler_config = f\"{start_time} {days}\"\n config = ConfigGenericSchedule(\"daily\", scheduler_config, None)\n return valid_daily_scheduler(config, config_context)\n\n path = config_context.path\n raise ConfigError(f\"Unknown scheduler at {path}: {schedule}\")\n\n\ndef valid_schedule(schedule, config_context):\n if isinstance(schedule, str):\n return schedule_config_from_string(schedule, config_context)\n\n if \"type\" not in schedule:\n return schedule_config_from_legacy_dict(schedule, config_context)\n\n schedule = ScheduleValidator().validate(schedule, config_context)\n return validate_generic_schedule_config(schedule, config_context)\n\n\ndef valid_daily_scheduler(config, config_context):\n \"\"\"Daily scheduler, accepts a time of day and an optional list of days.\"\"\"\n schedule_config = config.value\n time_string, days = pad_sequence(schedule_config.split(), 2)\n time_string = time_string or \"00:00:00\"\n time_spec = config_utils.valid_time(time_string, config_context)\n days = config_utils.valid_string(days or \"\", config_context)\n\n def valid_day(day):\n if day not in CONVERT_DAYS_INT:\n raise ConfigError(\n f\"Unknown day {day} at {config_context.path}\",\n )\n return CONVERT_DAYS_INT[day]\n\n original = f\"{time_string} {days}\"\n weekdays = {valid_day(day) for day in days or ()}\n return ConfigDailyScheduler(\n original,\n time_spec.hour,\n time_spec.minute,\n time_spec.second,\n weekdays,\n jitter=config.jitter,\n )\n\n\ndef normalize_weekdays(seq):\n return seq[6:7] + seq[:6]\n\n\ndef day_canonicalization_map():\n \"\"\"Build a map of weekday synonym to int index 0-6 inclusive.\"\"\"\n canon_map = dict()\n\n # 7-element lists with weekday names in order\n weekday_lists = [\n normalize_weekdays(calendar.day_name),\n normalize_weekdays(calendar.day_abbr),\n (\n \"u\",\n \"m\",\n \"t\",\n \"w\",\n \"r\",\n \"f\",\n \"s\",\n ),\n (\n \"su\",\n \"mo\",\n \"tu\",\n \"we\",\n \"th\",\n \"fr\",\n \"sa\",\n ),\n ]\n for day_list in weekday_lists:\n for day_name_synonym, day_index in zip(day_list, range(7)):\n canon_map[day_name_synonym] = day_index\n canon_map[day_name_synonym.lower()] = day_index\n canon_map[day_name_synonym.upper()] = day_index\n\n return canon_map\n\n\n# Canonicalize weekday names to integer indices\nCONVERT_DAYS_INT = day_canonicalization_map() # day name/abbrev => {0123456}\n\n\ndef month_canonicalization_map():\n \"\"\"Build a map of month synonym to int index 0-11 inclusive.\"\"\"\n canon_map = dict()\n\n # calendar stores month data with a useless element in front. cut it off.\n monthname_lists = (calendar.month_name[1:], calendar.month_abbr[1:])\n for month_list in monthname_lists:\n for key, value in zip(month_list, range(1, 13)):\n canon_map[key] = value\n canon_map[key.lower()] = value\n return canon_map\n\n\n# Canonicalize month names to integer indices\n# month name/abbrev => {0 <= k <= 11}\nCONVERT_MONTHS = month_canonicalization_map()\n\n\ndef build_groc_schedule_parser_re():\n \"\"\"Build a regular expression that matches this:\n\n (\"every\"|ordinal) (day) [\"of|in\" (monthspec)] ([\"at\"] HH:MM)\n\n ordinal - comma-separated list of \"1st\" and so forth\n days - comma-separated list of days of the week (for example,\n \"mon\", \"tuesday\", with both short and long forms being\n accepted); \"every day\" is equivalent to\n \"every mon,tue,wed,thu,fri,sat,sun\"\n monthspec - comma-separated list of month names (for example,\n \"jan\", \"march\", \"sep\"). If omitted, implies every month.\n You can also say \"month\" to mean every month, as in\n \"1,8th,15,22nd of month 09:00\".\n HH:MM - time of day in 24 hour time.\n\n This is a slightly more permissive version of Google App Engine's schedule\n parser, documented here:\n http://code.google.com/appengine/docs/python/config/cron.html#The_Schedule_Format\n \"\"\"\n\n # m|mon|monday|...|day\n DAY_VALUES = \"|\".join(list(CONVERT_DAYS_INT.keys()) + [\"day\"])\n\n # jan|january|...|month\n MONTH_VALUES = \"|\".join(list(CONVERT_MONTHS.keys()) + [\"month\"])\n\n DATE_SUFFIXES = \"st|nd|rd|th\"\n\n # every|1st|2nd|3rd (also would accept 3nd, 1rd, 4st)\n MONTH_DAYS_EXPR = r\"(?Pevery|((\\d+(%s),?)+))?\" % DATE_SUFFIXES\n DAYS_EXPR = r\"((?P((%s),?)+))?\" % DAY_VALUES\n MONTHS_EXPR = r\"((in|of)\\s+(?P((%s),?)+))?\" % MONTH_VALUES\n\n # [at] 00:00\n TIME_EXPR = r\"((at\\s+)?(?P