Repository: Yelp/Tron Branch: master Commit: 5779faaa0153 Files: 347 Total size: 2.5 MB Directory structure: gitextract_3urb6sob/ ├── .dockerignore ├── .github/ │ └── workflows/ │ ├── ci.yml │ └── security-review.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .pyautotest ├── .readthedocs.yaml ├── AGENTS.md ├── CODEOWNERS ├── LICENSE.txt ├── MANIFEST.in ├── Makefile ├── OWNERS ├── README.md ├── bin/ │ ├── generate_tron_tab_completion_cache │ ├── tronctl │ ├── tronctl_tabcomplete.sh │ ├── trond │ ├── tronfig │ ├── tronrepl │ ├── tronview │ └── tronview_tabcomplete.sh ├── contrib/ │ ├── migration_script.py │ ├── mock_patch_checker.py │ ├── namespace_cleanup.sh │ ├── patch-config-loggers.diff │ ├── sync-from-yelp-prod.sh │ └── sync_namespaces_jobs.py ├── debian/ │ ├── changelog │ ├── compat │ ├── control │ ├── copyright │ ├── docs │ ├── install │ ├── pycompat │ ├── pyversions │ ├── rules │ ├── tron.conffiles │ ├── tron.default │ ├── tron.dirs │ ├── tron.example │ ├── tron.links │ ├── tron.manpages │ ├── tron.postinst │ ├── tron.service │ ├── tron.upstart │ └── watch ├── dev/ │ ├── config/ │ │ ├── MASTER.yaml │ │ └── _manifest.yaml │ └── logging.conf ├── docs/ │ └── source/ │ ├── _static/ │ │ └── nature.css │ ├── command_context.rst │ ├── conf.py │ ├── config.rst │ ├── developing.rst │ ├── generated/ │ │ ├── modules.rst │ │ ├── tron.actioncommand.rst │ │ ├── tron.api.adapter.rst │ │ ├── tron.api.async_resource.rst │ │ ├── tron.api.auth.rst │ │ ├── tron.api.controller.rst │ │ ├── tron.api.requestargs.rst │ │ ├── tron.api.resource.rst │ │ ├── tron.api.rst │ │ ├── tron.command_context.rst │ │ ├── tron.commands.authentication.rst │ │ ├── tron.commands.backfill.rst │ │ ├── tron.commands.client.rst │ │ ├── tron.commands.cmd_utils.rst │ │ ├── tron.commands.display.rst │ │ ├── tron.commands.retry.rst │ │ ├── tron.commands.rst │ │ ├── tron.config.config_parse.rst │ │ ├── tron.config.config_utils.rst │ │ ├── tron.config.manager.rst │ │ ├── tron.config.rst │ │ ├── tron.config.schedule_parse.rst │ │ ├── tron.config.schema.rst │ │ ├── tron.config.static_config.rst │ │ ├── tron.core.action.rst │ │ ├── tron.core.actiongraph.rst │ │ ├── tron.core.actionrun.rst │ │ ├── tron.core.job.rst │ │ ├── tron.core.job_collection.rst │ │ ├── tron.core.job_scheduler.rst │ │ ├── tron.core.jobgraph.rst │ │ ├── tron.core.jobrun.rst │ │ ├── tron.core.recovery.rst │ │ ├── tron.core.rst │ │ ├── tron.eventbus.rst │ │ ├── tron.kubernetes.rst │ │ ├── tron.manhole.rst │ │ ├── tron.mcp.rst │ │ ├── tron.mesos.rst │ │ ├── tron.metrics.rst │ │ ├── tron.node.rst │ │ ├── tron.prom_metrics.rst │ │ ├── tron.rst │ │ ├── tron.scheduler.rst │ │ ├── tron.serialize.filehandler.rst │ │ ├── tron.serialize.rst │ │ ├── tron.serialize.runstate.dynamodb_state_store.rst │ │ ├── tron.serialize.runstate.rst │ │ ├── tron.serialize.runstate.shelvestore.rst │ │ ├── tron.serialize.runstate.statemanager.rst │ │ ├── tron.serialize.runstate.yamlstore.rst │ │ ├── tron.ssh.rst │ │ ├── tron.trondaemon.rst │ │ ├── tron.utils.collections.rst │ │ ├── tron.utils.crontab.rst │ │ ├── tron.utils.exitcode.rst │ │ ├── tron.utils.logreader.rst │ │ ├── tron.utils.observer.rst │ │ ├── tron.utils.persistable.rst │ │ ├── tron.utils.proxy.rst │ │ ├── tron.utils.queue.rst │ │ ├── tron.utils.rst │ │ ├── tron.utils.state.rst │ │ ├── tron.utils.timeutils.rst │ │ ├── tron.utils.trontimespec.rst │ │ ├── tron.utils.twistedutils.rst │ │ └── tron.yaml.rst │ ├── index.rst │ ├── jobs.rst │ ├── man/ │ │ ├── tronctl.1 │ │ ├── trond.8 │ │ ├── tronfig.1 │ │ └── tronview.1 │ ├── man_tronctl.rst │ ├── man_trond.rst │ ├── man_tronfig.rst │ ├── man_tronview.rst │ ├── overview.rst │ ├── sample_config.yaml │ ├── tools.rst │ ├── tron.yaml │ ├── tronweb.rst │ ├── tutorial.rst │ └── whats-new.rst ├── itest.sh ├── mypy.ini ├── osx-bdb.sh ├── package.json ├── pyproject.toml ├── requirements-dev-minimal.txt ├── requirements-dev.txt ├── requirements-docs.txt ├── requirements-minimal.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── testfiles/ │ └── MASTER.yaml ├── testifycompat/ │ ├── __init__.py │ ├── assertions.py │ ├── bin/ │ │ ├── __init__.py │ │ └── migrate.py │ └── fixtures.py ├── tests/ │ ├── __init__.py │ ├── actioncommand_test.py │ ├── api/ │ │ ├── __init__.py │ │ ├── adapter_test.py │ │ ├── auth_test.py │ │ ├── controller_test.py │ │ ├── requestargs_test.py │ │ └── resource_test.py │ ├── assertions.py │ ├── bin/ │ │ ├── __init__.py │ │ ├── action_runner_test.py │ │ ├── action_status_test.py │ │ ├── check_tron_jobs_test.py │ │ ├── get_tron_metrics_test.py │ │ └── recover_batch_test.py │ ├── command_context_test.py │ ├── commands/ │ │ ├── __init__.py │ │ ├── backfill_test.py │ │ ├── client_test.py │ │ ├── cmd_utils_test.py │ │ ├── display_test.py │ │ └── retry_test.py │ ├── config/ │ │ ├── __init__.py │ │ ├── config_parse_test.py │ │ ├── config_utils_test.py │ │ ├── manager_test.py │ │ └── schedule_parse_test.py │ ├── core/ │ │ ├── __init__.py │ │ ├── action_test.py │ │ ├── actiongraph_test.py │ │ ├── actionrun_test.py │ │ ├── job_collection_test.py │ │ ├── job_scheduler_test.py │ │ ├── job_test.py │ │ ├── jobgraph_test.py │ │ ├── jobrun_test.py │ │ └── recovery_test.py │ ├── data/ │ │ ├── logging.conf │ │ └── test_config.yaml │ ├── eventbus_test.py │ ├── kubernetes_test.py │ ├── mcp_reconfigure_test.py │ ├── mcp_test.py │ ├── mesos_test.py │ ├── metrics_test.py │ ├── mocks.py │ ├── node_test.py │ ├── sandbox.py │ ├── scheduler_test.py │ ├── serialize/ │ │ ├── __init__.py │ │ ├── filehandler_test.py │ │ └── runstate/ │ │ ├── __init__.py │ │ ├── dynamodb_state_store_test.py │ │ ├── shelvestore_test.py │ │ ├── statemanager_test.py │ │ └── yamlstore_test.py │ ├── ssh_test.py │ ├── test_id_rsa │ ├── test_id_rsa.pub │ ├── testingutils.py │ ├── tools/ │ │ └── sync_tron_state_from_k8s_test.py │ ├── trond_test.py │ ├── trondaemon_test.py │ └── utils/ │ ├── __init__.py │ ├── collections_test.py │ ├── crontab_test.py │ ├── logreader_test.py │ ├── observer_test.py │ ├── proxy_test.py │ ├── shortOutputTest.txt │ ├── state_test.py │ ├── timeutils_test.py │ └── trontimespec_test.py ├── tools/ │ ├── action_dag_diagram.py │ ├── compress_json.py │ ├── inspect_serialized_state.py │ ├── migration/ │ │ ├── migrate_config_0.2_to_0.3.py │ │ ├── migrate_config_0.5.1_to_0.5.2.py │ │ ├── migrate_state.py │ │ └── migrate_state_1.3.15_to_1.4.0.py │ ├── pickles_to_json.py │ └── sync_tron_state_from_k8s.py ├── tox.ini ├── tron/ │ ├── __init__.py │ ├── actioncommand.py │ ├── api/ │ │ ├── __init__.py │ │ ├── adapter.py │ │ ├── async_resource.py │ │ ├── auth.py │ │ ├── controller.py │ │ ├── requestargs.py │ │ └── resource.py │ ├── bin/ │ │ ├── action_runner.py │ │ ├── action_status.py │ │ ├── check_tron_datastore_staleness.py │ │ ├── check_tron_jobs.py │ │ ├── get_tron_metrics.py │ │ └── recover_batch.py │ ├── command_context.py │ ├── commands/ │ │ ├── __init__.py │ │ ├── authentication.py │ │ ├── backfill.py │ │ ├── client.py │ │ ├── cmd_utils.py │ │ ├── display.py │ │ └── retry.py │ ├── config/ │ │ ├── __init__.py │ │ ├── config_parse.py │ │ ├── config_utils.py │ │ ├── manager.py │ │ ├── schedule_parse.py │ │ ├── schema.py │ │ ├── static_config.py │ │ └── tronfig_schema.json │ ├── core/ │ │ ├── __init__.py │ │ ├── action.py │ │ ├── actiongraph.py │ │ ├── actionrun.py │ │ ├── job.py │ │ ├── job_collection.py │ │ ├── job_scheduler.py │ │ ├── jobgraph.py │ │ ├── jobrun.py │ │ └── recovery.py │ ├── default_config.yaml │ ├── eventbus.py │ ├── kubernetes.py │ ├── logging.conf │ ├── manhole.py │ ├── mcp.py │ ├── mesos.py │ ├── metrics.py │ ├── node.py │ ├── prom_metrics.py │ ├── scheduler.py │ ├── serialize/ │ │ ├── __init__.py │ │ ├── filehandler.py │ │ └── runstate/ │ │ ├── __init__.py │ │ ├── dynamodb_state_store.py │ │ ├── shelvestore.py │ │ ├── statemanager.py │ │ └── yamlstore.py │ ├── ssh.py │ ├── trondaemon.py │ ├── utils/ │ │ ├── __init__.py │ │ ├── collections.py │ │ ├── crontab.py │ │ ├── exitcode.py │ │ ├── logreader.py │ │ ├── observer.py │ │ ├── persistable.py │ │ ├── proxy.py │ │ ├── queue.py │ │ ├── state.py │ │ ├── timeutils.py │ │ ├── trontimespec.py │ │ └── twistedutils.py │ └── yaml.py ├── tronweb/ │ ├── coffee/ │ │ ├── actionrun.coffee │ │ ├── config.coffee │ │ ├── dashboard.coffee │ │ ├── graph.coffee │ │ ├── job.coffee │ │ ├── models.coffee │ │ ├── navbar.coffee │ │ ├── nodes.coffee │ │ ├── routes.coffee │ │ ├── timeline.coffee │ │ └── views.coffee │ ├── css/ │ │ ├── codemirror.css │ │ ├── tronweb.less │ │ └── whhg.css │ ├── fonts/ │ │ └── SIL OFL Font License WebHostingHub Glyphs.txt │ ├── index.html │ └── js/ │ ├── backbone-min.js │ ├── codemirror.js │ ├── plugins.js │ ├── underscore-min.js │ ├── underscore.extra.js │ ├── underscore.string.js │ └── yaml.js ├── tronweb_tests/ │ ├── SpecRunner.html │ ├── spec/ │ │ └── README │ └── tests/ │ ├── actionrun_test.coffee │ ├── dashboard_test.coffee │ ├── navbar_test.coffee │ ├── routes_test.coffee │ └── timeline_test.coffee └── yelp_package/ ├── extra_requirements_yelp.txt └── jammy/ └── Dockerfile ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ .tox .git .idea ================================================ FILE: .github/workflows/ci.yml ================================================ --- name: tron-ci on: push: branches: - master tags: - v*.* pull_request: release: jobs: tox: runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: toxenv: - py310,docs steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: '3.10' # GHA won't setup tox for us - run: pip install tox==3.2 # there are no pre-built wheels for bsddb3, so we need to install # its dpkg dependencies so that we can build a wheel when we're # creating our env. Once we get rid of bsddb3 as a Python dependency, # then we can also get rid of this dpkg - run: sudo apt-get install --quiet --assume-yes libdb5.3-dev # we explictly attempt to import the C extensions for some PyYAML # functionality, so we need the LibYAML bindings provided by this # package - run: sudo apt-get install --quiet --assume-yes libyaml-dev - run: tox -e ${{ matrix.toxenv }} build_debs: runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: dist: - jammy steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: '3.10' # Update package lists to ensure we have the latest information - run: sudo apt-get update # the container provided by GitHub doesn't include utilities # needed for dpkg building, so we need to install `devscripts` # to bring those in - run: sudo apt-get install --quiet --assume-yes devscripts - run: make itest_${{ matrix.dist }} - uses: actions/upload-artifact@v4 with: name: deb-${{ matrix.dist }} path: dist/tron_*.deb cut_release: runs-on: ubuntu-22.04 needs: build_debs steps: - uses: actions/checkout@v2 - run: mkdir -p dist/ - uses: actions/download-artifact@v4 with: name: deb-jammy path: dist/ - name: Release uses: softprops/action-gh-release@v1 if: startsWith(github.ref, 'refs/tags/v') with: generate_release_notes: true files: | dist/tron_*.deb fail_on_unmatched_files: true ================================================ FILE: .github/workflows/security-review.yml ================================================ # Managed by terraform, do not edit manually name: Security Review permissions: pull-requests: write contents: read id-token: write on: pull_request: jobs: security: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} fetch-depth: 2 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_DEV_ACCOUNT_ID }}:role/security-review-bot aws-region: us-west-2 - uses: anthropics/claude-code-security-review@0c6a49f1fa56a1d472575da86a94dbc1edb78eda with: comment-pr: true claude-api-key: "github-actions" claude-model: "us.anthropic.claude-opus-4-6-v1" run-every-commit: true env: CLAUDE_CODE_USE_BEDROCK: "1" AWS_REGION: "us-west-2" ================================================ FILE: .gitignore ================================================ dist build MANIFEST tron.egg-info *.pyc *._* *.swp *.swo docs/_build/ .idea .vscode .fleet tron.iml docs/images/ *.dot tronweb/js/cs/*.js yarn.lock tronweb_tests/spec/*.js tronweb_tests/lib/ .tox .tox-indocker tron.iml __pycache__/ .pytest_cache/ tron_state tron.lock manhole.sock manhole.sock.lock node_modules/ # Example cluster example-cluster/config example-cluster/MASTER.* example-cluster/tron-repl.lock example-cluster/tron_state* example-cluster/manhole.sock* example-cluster/_events/ *.stdout *.stderr dev/manhole.sock.lock dev/tron.pid dev/_events/ # Generated debian artifacts debian/.debhelper/ debian/debhelper-build-stamp debian/files debian/tron debian/tron.debhelper.log debian/tron.postinst.debhelper debian/tron.postrm.debhelper debian/tron.preinst.debhelper debian/tron.prerm.debhelper debian/tron.substvars ================================================ FILE: .pre-commit-config.yaml ================================================ --- default_language_version: python: python3.10 repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer exclude: CHANGELOG.md - id: check-docstring-first - id: check-json - id: check-yaml - id: requirements-txt-fixer - id: fix-encoding-pragma args: [--remove] - id: pretty-format-json args: [--autofix, --indent, '4', --no-sort-keys] - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: - id: flake8 exclude: ^docs/source/conf.py$ - repo: https://github.com/asottile/reorder_python_imports rev: v1.9.0 hooks: - id: reorder-python-imports args: [--py3-plus] - repo: https://github.com/asottile/pyupgrade rev: v3.20.0 hooks: - id: pyupgrade args: [--py39-plus] - repo: local hooks: - id: patch-enforce-autospec name: mock.patch enforce autospec description: | This hook ensures all mock.patch invocations specify an autospec entry: contrib/mock_patch_checker.py language: script files: ^tests/.*\.py$ - repo: http://github.com/psf/black rev: 22.3.0 hooks: - id: black args: [--target-version, py310] ================================================ FILE: .pyautotest ================================================ test_runner_name: "testify" ================================================ FILE: .readthedocs.yaml ================================================ # Read the Docs configuration file for Sphinx projects # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required version: 2 # RTD defaults as of 2023-11-08 build: os: ubuntu-22.04 tools: python: "3.10" # You can also specify other tool versions: # nodejs: "20" # rust: "1.70" # golang: "1.20" # Also provide downloadable zip formats: [htmlzip] # Build documentation in the "docs/" directory with Sphinx sphinx: configuration: docs/source/conf.py # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs # builder: "dirhtml" # Fail on all warnings to avoid broken references # fail_on_warning: true # Optionally build your docs in additional formats such as PDF and ePub # formats: # - pdf # - epub # Optional but recommended, declare the Python requirements required # to build your documentation # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html python: install: - requirements: requirements-docs.txt ================================================ FILE: AGENTS.md ================================================ # AGENTS.md This file provides guidance to AI coding agents working with code in this repository. ## What is Tron? Tron is Yelp's centralized batch job scheduling system—a distributed alternative to cron for managing periodic batch processes across a cluster. ### Core Concepts **Jobs** are DAGs (directed acyclic graphs) defined in YAML configuration. Each job contains one or more **Actions** (individual commands) with dependency relationships between them. **JobRuns** are instances of a job execution. If a job runs daily, each execution creates a new JobRun containing **ActionRuns** for each action. Tron tracks each run independently. ### Lifecycle ```mermaid flowchart LR A[Startup] --> B[Restore state from DynamoDB] B --> C[Schedule jobs] C --> D[Track state changes] D --> E[Save state to DynamoDB] E --> C ``` ### State Persistence Tron persists all job and run state to DynamoDB. 101 consecutive save errors triggers intentional crash (prevents running degraded). ```mermaid flowchart LR A[State change] --> B[Buffer] B -->|buffer full| C[Save queue] C --> D[Pop from queue] D --> E[Delete existing entries] E --> F[Partition into chunks] F --> G[Batch write to DynamoDB] G -->|failure| C ``` ### Execution Backends - **Kubernetes**: Primary execution backend - **SSH**: Legacy backend for remote command execution, do not extend - **Mesos**: Deprecated, do not extend ## Project Structure ``` tron/ ├── core/ # Job, Action, JobRun, scheduling, dependency graphs ├── config/ # YAML config parsing and schema definitions ├── serialize/ # State persistence (DynamoDB, shelve backends) ├── api/ # REST API endpoints and adapters ├── kubernetes.py # Kubernetes execution backend └── mesos.py # Deprecated - do not modify tronweb/ # Web UI (CoffeeScript/Backbone.js) bin/ # CLI: trond, tronctl, tronview, tronfig ``` ## Testing Tox manages the virtualenv in `.tox/py310/`. Use `make test` for the full suite, or iterate with pytest directly: ```bash .tox/py310/bin/pytest tests/path/to/test.py -x ``` ## Development Guardrails ### High-risk areas **DynamoDB/Persistence changes:** - Pickle deserialization is still active—deleting or renaming persisted classes/fields breaks restore - Reverting changes that add new persisted fields is NOT safe - Writes batch 8 partitions at a time; large jobs needing more can be partially written if a later batch fails **Job/action schema changes** require updates in two repos: - Tron: `tron/config/schema.py`, `tron/core/action.py` (including `from_json`/`to_json`) - PaaSTA: `paasta_tools/cli/schemas/tron_schema.json`, `paasta_tools/tron_tools.py` - Plus: tests in both repos, and any code that consumes the new field **MASTER config changes**: - `tron/config/schema.py` — Add field to config object - `tron/config/config_parse.py` — Add default value and validator - Plus: tests, and any code that consumes the new config value Reverting config changes is risky: new params get written to MASTER.yaml on disk, so reverting code requires manual config cleanup on servers. ### Do not modify - `tron/mesos.py` — Deprecated - `tron/ssh.py` - Deprecated - `tron/node.py` - Deprecated - DynamoDB schema without approval ================================================ FILE: CODEOWNERS ================================================ # NOTE: "we" in this file will refer to the Compute Infrastructure team at Yelp * @Yelp/paasta # # prevent cheeky modifications :) CODEOWNERS @Yelp/paasta ================================================ FILE: LICENSE.txt ================================================ Copyright 2010-2012 Yelp Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: MANIFEST.in ================================================ include *.txt include *.md include Makefile include tron/default_config.yaml include tron/logging.conf include tron/named_config_template.yaml recursive-include tests *.py *.yaml recursive-include docs *.rst *.yaml *.1 *.8 recursive-include tronweb * recursive-exclude tronweb *.coffee ================================================ FILE: Makefile ================================================ # Edit this release and run "make release" RELEASE=3.10.0 SHELL=/bin/bash DOCKER_RUN = docker run -t -v $(CURDIR):/work:rw -v $(CURDIR)/.tox-indocker:/work/.tox:rw UID:=$(shell id -u) GID:=$(shell id -g) ifeq ($(findstring .yelpcorp.com,$(shell hostname -f)), .yelpcorp.com) PAASTA_ENV ?= YELP else PAASTA_ENV ?= $(shell hostname --fqdn) endif NOOP = true ifeq ($(PAASTA_ENV),YELP) export PIP_INDEX_URL ?= http://169.254.255.254:20641/$*/simple/ ADD_MISSING_DEPS_MAYBE:=-diff --unchanged-line-format= --old-line-format= --new-line-format='%L' ./requirements.txt ./yelp_package/extra_requirements_yelp.txt >> ./requirements.txt else export PIP_INDEX_URL ?= https://pypi.python.org/simple ADD_MISSING_DEPS_MAYBE:=$(NOOP) endif .PHONY : all clean tests docs dev -usage: @echo "make test - Run tests" @echo "make deb_jammy - Generate jammy deb package" @echo "make itest_jammy - Run tests and integration checks" @echo "make _itest_jammy - Run only integration checks" @echo "make release - Prepare debian info for new release" @echo "make clean - Get rid of scratch and byte files" @echo "make dev - Get a local copy of trond running in debug mode in the foreground" docker_%: @echo "Building docker image for $*" [ -d dist ] || mkdir -p dist cd ./yelp_package/$* && docker build --build-arg PIP_INDEX_URL=${PIP_INDEX_URL} -t tron-builder-$* . deb_%: clean docker_% coffee_% @echo "Building deb for $*" # backup these files so we can temp modify them cp requirements.txt requirements.txt.old $(ADD_MISSING_DEPS_MAYBE) $(DOCKER_RUN) -e PIP_INDEX_URL=${PIP_INDEX_URL} tron-builder-$* /bin/bash -c ' \ dpkg-buildpackage -d && \ mv ../*.deb dist/ && \ rm -rf debian/tron \ ' # restore the backed up files mv requirements.txt.old requirements.txt coffee_%: docker_% @echo "Building tronweb" $(DOCKER_RUN) tron-builder-$* /bin/bash -c ' \ rm -rf tronweb/js/cs && \ mkdir -p tronweb/js/cs && \ coffee -o tronweb/js/cs/ -c tronweb/coffee/ \ ' test: tox -e py310 test_in_docker_%: docker_% $(DOCKER_RUN) tron-builder-$* python3.10 -m tox -vv -e py310 tox_%: tox -e $* _itest_%: $(DOCKER_RUN) ubuntu:$* /work/itest.sh debitest_%: deb_% _itest_% @echo "Package for $* looks good" itest_%: debitest_% @echo "itest $* OK" dev: SSH_AUTH_SOCK=$(SSH_AUTH_SOCK) .tox/py310/bin/trond --debug --working-dir=dev -l logging.conf --host=0.0.0.0 example_cluster: tox -e example-cluster yelpy: .tox/py310/bin/pip install -r yelp_package/extra_requirements_yelp.txt # 1. Bump version at the top of this file # 2. `make release` VERSION = $(firstword $(subst -, ,$(RELEASE) )) LAST_COMMIT_MSG = $(shell git log -1 --pretty=%B | sed -e 's/\x27/"/g') release: @if [[ "$$(git status --porcelain --untracked-files=no :^/Makefile)" != '' ]]; then echo "Error: Working directory is not clean; only changes to Makefile are allowed when cutting a release."; exit 1; fi $(eval untracked_files_tmpfile=$(shell mktemp)) git status --porcelain --untracked-files=all :^./Makefile > $(untracked_files_tmpfile) @if [[ "$$(git status --porcelain --untracked-files=normal :/docs/source/generated)" != '' ]]; then echo "Error: Untracked files found in docs/source/generated."; exit 1; fi @if existing_sha=$$(git rev-parse --verify --quiet v$(VERSION)); then echo "Error: tag v$(VERSION) exists and points at $$existing_sha"; exit 1; fi @read upstream_master junk <<<"$$(git ls-remote -h origin master)" && if ! git merge-base --is-ancestor $$upstream_master HEAD; then echo "Error: HEAD is missing commits from origin/master ($$upstream_master)."; exit 1; fi dch -v $(RELEASE) --distribution jammy --changelog ./debian/changelog $$'$(VERSION) tagged with \'make release\'\rCommit: $(LAST_COMMIT_MSG)' sed -i -e "s/__version__ = .*/__version__ = \"$(VERSION)\"/" ./tron/__init__.py make docs || true git add ./Makefile ./debian/changelog ./tron/__init__.py ./docs/source/generated/ git commit -m "Released $(RELEASE) via make release" if [[ "$$(git status --porcelain --untracked-files=all)" != "$$(<$(untracked_files_tmpfile))" ]]; then echo "Error: automatic git commit left some files uncommitted. Fix the git commit command in ./Makefile to include any automatically generated files that it is currently missing."; exit 1; fi git tag v$(VERSION) git push --atomic origin master v$(VERSION) docs: tox -r -e docs man: which $(SPHINXBUILD) >/dev/null && $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(DOCS_DIR) $(DOCS_DIR)/source/man || true @echo @echo "Build finished. The manual pages are in $(DOCS_BUILDDIR)/source/man." clean: rm -rf tronweb/js/cs find . -name '*.pyc' -delete ================================================ FILE: OWNERS ================================================ --- teams: - Compute Infra ================================================ FILE: README.md ================================================ Tron - Batch Scheduling System ============================== [![Build Status](https://github.com/Yelp/Tron/actions/workflows/ci.yml/badge.svg?query=branch%3Amaster)](https://github.com/Yelp/Tron/actions/workflows/ci.yml) [![Documentation Status](https://readthedocs.org/projects/tron/badge/?version=latest)](http://tron.readthedocs.io/en/latest/?badge=latest) Tron is a centralized system for managing periodic batch processes across a cluster. If you find [cron](http://en.wikipedia.org/wiki/Cron) or [fcron](http://fcron.free.fr/) to be insufficient for managing complex work flows across multiple computers, Tron might be for you. Install with: > sudo pip install tron Or look at the [tutorial](http://tron.readthedocs.io/en/latest/tutorial.html). The full documentation is available [on ReadTheDocs](http://tron.readthedocs.io/en/latest/). Versions / Roadmap ------------------ Tron is changing and under active development. It is being transformed from an ssh-based execution engine to be compatible with running on [Kubernetes ](https://kubernetes.io/docs/concepts/overview/). Tron development is specifically targeting Yelp's needs and not designed to be a general solution for other companies. Contributing ------------ Read [Working on Tron](http://tron.readthedocs.io/en/latest/developing.html) and start sending pull requests! Any issues should be posted [on Github](http://github.com/Yelp/Tron/issues). BerkeleyDB on Mac OS X ---------------------- $ brew install berkeley-db $ export BERKELEYDB_DIR=$(brew --cellar)/berkeley-db/ $ export YES_I_HAVE_THE_RIGHT_TO_USE_THIS_BERKELEY_DB_VERSION=1 ================================================ FILE: bin/generate_tron_tab_completion_cache ================================================ #!/usr/bin/env python """ print a list of all the tron jobs, to be saved as a cache for tab completion """ import argcomplete from tron.commands import cmd_utils from tron.commands.client import Client def main(): parser = cmd_utils.build_option_parser() argcomplete.autocomplete(parser) args = parser.parse_args() cmd_utils.load_config(args) client = Client(args.server) for job in client.jobs(include_job_runs=True, include_action_runs=True): print(job["name"]) for run in job["runs"]: print(run["id"]) for action in run["runs"]: print(action["id"]) if __name__ == "__main__": main() ================================================ FILE: bin/tronctl ================================================ #!/usr/bin/env python """Tron Control Part of the command line interface to the tron daemon. Provides the interface to controlling jobs and runs. """ import argparse import asyncio import datetime import logging import pprint import sys from collections import defaultdict from collections.abc import Callable from collections.abc import Generator from typing import Any from urllib.parse import urljoin import argcomplete # type: ignore from tron import __version__ from tron.commands import client from tron.commands import cmd_utils from tron.commands.backfill import BackfillRun from tron.commands.backfill import confirm_backfill from tron.commands.backfill import DEFAULT_MAX_PARALLEL_RUNS from tron.commands.backfill import get_date_range from tron.commands.backfill import LIMIT_MAX_PARALLEL_RUNS from tron.commands.backfill import print_backfill_cmds from tron.commands.backfill import print_backfill_runs_table from tron.commands.backfill import run_backfill_for_date_range from tron.commands.client import RequestError from tron.commands.client import TronObjectIdentifier from tron.commands.cmd_utils import COLOR_YELLOW from tron.commands.cmd_utils import ExitCode from tron.commands.cmd_utils import suggest_possibilities from tron.commands.cmd_utils import tron_jobs_completer from tron.commands.cmd_utils import warning_output from tron.commands.retry import parse_deps_timeout from tron.commands.retry import print_retries_table from tron.commands.retry import retry_actions from tron.commands.retry import RetryAction COMMAND_HELP = ( ( "start", "job name, job run id, or action id", "Start the selected job, job run, or action. Creates a new job run if starting a job.", ), ( "rerun", "job run id", "Start a new job run with the same start time command context as the given job run.", ), ( "retry", "action id", "Re-run a job action within an existing job run. Uses latest code/config except the command by default. Add --use-latest-command to use the latest command.", ), ("recover", "action id", "Ask Tron to start tracking an UNKNOWN action run again"), ("cancel", "job run id", "Cancel the selected job run."), ( "backfill", "job name", "Start job runs for a particular date range", ), ( "disable", "job name", "Disable selected job and cancel any outstanding runs. WARNING: you *must* disable the job in yelpsoa-configs to guarantee it will not be re-enabled.", ), ("enable", "job name", "Enable the selected job and schedule the next run"), ( "fail", "job run or action id", "Mark an UNKNOWN job or action as failed. Does not publish action triggers.", ), ( "success", "job run or action id", "Mark an UNKNOWN job or action as having succeeded. Will publish action triggers.", ), ( "skip", "action id", "Skip a failed action, unblocks dependent actions. Does *not* publish action triggers.", ), ( "skip-and-publish", "action id", "Skip a failed action, unblocks dependent actions. *Does* publish action triggers.", ), ("stop", "action id", "Stop the action run (SIGTERM)"), ("kill", "action id", "Force kill the action run (SIGKILL)"), ("move", "job name", "Rename a job"), ("publish", "trigger id", "Publish actionrun trigger to kick off downstream jobs"), ("discard", "trigger id", "Discard existing actionrun trigger"), ("version", None, "Print tron client and server versions"), ) log = logging.getLogger("tronctl") def parse_date(date_string): return datetime.datetime.strptime(date_string, "%Y-%m-%d") def parse_cli(): parser = cmd_utils.build_option_parser() subparsers = parser.add_subparsers(dest="command", title="commands", help="Tronctl command to run", required=True) cmd_parsers = {} for cmd_name, id_help_text, desc in COMMAND_HELP: cmd_parsers[cmd_name] = subparsers.add_parser(cmd_name, help=desc, description=desc) if id_help_text: cmd_parsers[cmd_name].add_argument( "id", nargs="*", help=id_help_text ).completer = cmd_utils.tron_jobs_completer # HACK: this is slightly funky since we already add --verbose in cmd_utils.build_option_parser(), # but that requires something like tronctl --verbose start JOB rather than tronctl start -v JOB cmd_parsers[cmd_name].add_argument( "-v", "--verbose", action="count", help="Verbose logging", default=None, ) # start cmd_parsers["start"].add_argument( "--run-date", type=parse_date, dest="run_date", help="What the run-date should be set to", ) # backfill backfill_parser = cmd_parsers["backfill"] mutex_dates_group = backfill_parser.add_mutually_exclusive_group(required=True) mutex_dates_group.add_argument( "--start-date", type=parse_date, dest="start_date", help="First run-date to backfill", ) backfill_parser.add_argument( "--end-date", type=parse_date, dest="end_date", help=( "Last run-date to backfill (note: many jobs operate on date-1), " "assuming --start-date is set. This date is inclusive. Defaults to today." ), ) backfill_parser.add_argument( "--descending", action="store_true", default=False, help=( "If set, backfill from end date to start date. Otherwise, " "the default is to backfill from start date to end date." ), ) mutex_dates_group.add_argument( "-d", "--dates", type=lambda v: [parse_date(date_str.strip()) for date_str in v.split(",")], dest="dates", help=( "List of comma-separated dates to run backfills on. " "Backfills will be executed for dates in the order they are presented." ), ) backfill_parser.add_argument( "-P", "--max-parallel", type=int, dest="max_parallel", default=DEFAULT_MAX_PARALLEL_RUNS, help=( "The max number of dates that can be backfilled in parallel. " "Before setting, consider how much in resources your job needs. " "If it needs a lot, keep this number low, because there may not be " "enough resources in the cluster too satisfy the demand, which can " "adversely affect other jobs. " "The default is %(default)s." ), ) backfill_parser.add_argument( "--fail-on-error", dest="fail_on_error", action="store_true", default=False, help=( "If set, the overall backfill will fail immediately if a backfill " "for a single date fails. All in-progress backfills will cancelled. " "If a single backfill is still considered successful it was otherwise " "cancelled or skipped by the user. " "By default, individual backfill failures are ignored." ), ) backfill_parser.add_argument( "--dry-run", action="store_true", default=False, help="Prints the equivalent `tronctl start` commands for the backfill", ) # retry retry_parser = cmd_parsers["retry"] retry_parser.add_argument( "--use-latest-command", action="store_true", default=False, help="Use the latest command in tronfig rather than the original command when the action run was created", ) retry_parser.add_argument( "--wait-for-deps", type=parse_deps_timeout, default=0, dest="deps_timeout", help=( "Max duration to wait for upstream dependencies (upstream triggers " "and/or same job actions) before attempting to retry. " "If all dependencies are not done when the timeout expires, " "this command will exit with an error, and the action will NOT be retried. " "Must be either an int number of seconds, a human-readable/" "pytimeparse-parsable string, or 'infinity' to wait forever. " "Defaults to 0 (don't wait)." ), ) argcomplete.autocomplete(parser) args = parser.parse_args() return args def request(url: str, data: dict[str, Any], headers=None, method=None) -> bool: # We want every tronctl request to be attributable response = client.request(url, data=data, headers=headers, method=method, user_attribution=True) if response.error: print(f"Error: {response.content}") return False print(response.content.get("result", "OK")) return True def event_publish(args): for event in args.id: # trying to publish a job run/action run id will likely print multiple warnings # since the conditions are somewhat overlapping - only print the first one warning_printed = False split_event = event.split(".") # first, let's try to catch folks trying to publish a job run or action run id as a trigger # these will look something like NAMESPACE.JOB_NAME.RUN_NUMBER or NAMESPACE.JOB_NAME.RUN_NUMBER.ACTION_NAME # i.e., if the 3rd element is an integer, it's one of these if len(split_event) >= 3: try: int(split_event[2]) print( warning_output( f"\nWarning: the event id '{event}' looks like a job run or action run id rather than a trigger id!", color=COLOR_YELLOW, ) ) print( warning_output( "This is almost certainly incorrect and you want something like `tronctl publish $SERVICE.$JOB_NAME.$ACTION_NAME.$TRIGGER_NAME.$TRIGGER_VALUE`", color=COLOR_YELLOW, ) ) warning_printed = True except ValueError: pass if len(split_event) != 5 and not warning_printed: print( warning_output( f"\nWarning: '{event}' is too {'long' if len(split_event) > 5 else 'short'} and does not match the expected trigger format!", color=COLOR_YELLOW, ) ) print( warning_output( "This is almost certainly incorrect and you want something like `tronctl publish $SERVICE.$JOB_NAME.$ACTION_NAME.$TRIGGER_NAME.$TRIGGER_VALUE`", color=COLOR_YELLOW, ) ) yield request( urljoin(args.server, "/api/events"), dict(command="publish", event=event), ) def event_discard(args): for event in args.id: yield request( urljoin(args.server, "/api/events"), dict(command="discard", event=event), ) def _get_triggers_for_action(server: str, action_identifier: str) -> tuple[str, ...] | None: try: namespace, job_name, run_number, action_name = action_identifier.split(".") except ValueError: print( f"Unable to fully decompose {action_identifier}: expected an identifier of the form (namespace).(job).(run).(action)" ) return None trigger_response = client.request( uri=urljoin( server, f"/api/jobs/{namespace}.{job_name}/{run_number}/{action_name}", ), ) if trigger_response.error: print(f"Unable to fetch downstream triggers for {action_identifier}: {trigger_response.error}") return None # triggers are returned by the API as comma-separated values with a space after every comma, which is # not automation-friendly - thus the non-standard multi-character split triggers = trigger_response.content.get("trigger_downstreams", "").split(", ") # the API will return an empty string for actions with no triggers to emit, but splitting '' yields [''], # so we want to make sure that we return an empty iterable in this case return tuple(f"{namespace}.{job_name}.{action_name}.{trigger}" for trigger in triggers if trigger) def skip_and_publish(server: str, tron_id: TronObjectIdentifier, identifier: str) -> bool: all_success = True print(f"Skipping {identifier}...") if request( url=urljoin(server, tron_id.url), data={"command": "skip"}, ): print(f"Successfully skipped {identifier}.") print(f"\nFetching triggers to publish for {identifier}...") # a single action can have 0..N triggers to publish and these can be arbitrarily named, so we need to # query the API and figure out what triggers exist triggers = _get_triggers_for_action(server=server, action_identifier=identifier) if triggers is None: print(f"\nEncountered error getting triggers to publish for {identifier}!") return False elif not triggers: print(f"{identifier} has no triggers to publish - just skipping instead.") # TODO: should we check this up-front and refuse to skip if there are no triggers that will be # published rather than carry on under the assumption that the user copy-pasted/typo'd the identifier? return True else: # TODO: this loop should use event_publish(), but we'd need to refactor how the CLI works and stop passing # around the full set of args everywhere to do so print("\nTriggers to publish:") print("\n".join(f" * {trigger}" for trigger in triggers) + "\n") for trigger in triggers: print(f"Publishing trigger {trigger}...") if not request( url=urljoin(server, "/api/events"), data={"command": "publish", "event": trigger}, ): print( f"Failed to publish trigger {trigger} - you may want to retry this command or manually publish the trigger!" ) all_success = False else: print(f"\nFailed to skip {identifier}!") return False return all_success def control_objects(args: argparse.Namespace): tron_client = client.Client(args.server, user_attribution=True) url_index = tron_client.index() for identifier in args.id: try: tron_id = client.get_object_type_from_identifier( url_index, identifier, ) except ValueError as e: possibilities = list( tron_jobs_completer(prefix="", client=tron_client), ) suggestions = suggest_possibilities( word=identifier, possibilities=possibilities, ) raise SystemExit(f"Error: {e}{suggestions}") if args.command == "skip-and-publish": # this command is more of a pseudo-command - skip and publish are handled in two different resources # and changing the API would be painful, so instead we call skip + publish separately from the client # (i.e., this file) to implement this functionality yield skip_and_publish(args.server, tron_id, identifier) else: data = dict(command=args.command) if args.command == "start" and args.run_date: data["run_time"] = str(args.run_date) yield request(urljoin(args.server, tron_id.url), data) # NOTE: ideally we'd add this message in the JobController handle_command() function, but having the API return terminal escape codes # sounds like a bad idea, so we're doing it here instead if args.command == "disable": print( warning_output( "WARNING: jobs disabled with tronctl disable are *NOT* guaranteed to stay disabled. You must disable the job in yelpsoa-configs to guarantee it will not be re-enabled." ) ) def retry(args): if args.deps_timeout != RetryAction.NO_TIMEOUT: deps_timeout_str = "forever" # timeout = -1 (RetryAction.WAIT_FOREVER) if args.deps_timeout > 0: deps_timeout_str = "up to " + str(datetime.timedelta(seconds=args.deps_timeout)) print( f"We will wait {deps_timeout_str} for all upstream triggers to be published " "and required actions to finish successfully before issuing retries for the " "following actions:" ) print() pprint.pprint(args.id) print() retries = retry_actions(args.server, args.id, args.use_latest_command, args.deps_timeout) print_retries_table(retries) yield all([r.succeeded for r in retries]) def move(args): try: old_name = args.id[0] new_name = args.id[1] except IndexError as e: raise SystemExit(f"Error: Move command needs two arguments.\n{e}") tron_client = client.Client(args.server, user_attribution=True) url_index = tron_client.index() job_index = url_index["jobs"] if old_name not in job_index.keys(): raise SystemExit(f"Error: {old_name} doesn't exist") if new_name in job_index.keys(): raise SystemExit(f"Error: {new_name} exists already") data = dict(command="move", old_name=old_name, new_name=new_name) yield request(urljoin(args.server, "/api/jobs"), data) def backfill(args): if not args.id: print("Error: must provide at least one id argument") yield False if args.max_parallel > LIMIT_MAX_PARALLEL_RUNS: raise SystemExit( f"The flag --max-parallel exceeds the allowed limit of {LIMIT_MAX_PARALLEL_RUNS}. " + "Please reach out to the Tron team if you need to run backfills with higher limits." ) if args.start_date: if args.end_date is None: args.end_date = datetime.datetime.today() dates = get_date_range(args.start_date, args.end_date, descending=args.descending) else: dates = args.dates date_strs = [d.date().isoformat() for d in dates] job_name = args.id[0] if args.dry_run: print_backfill_cmds(job_name, date_strs) yield True else: if confirm_backfill(job_name, date_strs): loop = asyncio.get_event_loop() try: backfill_runs = loop.run_until_complete( run_backfill_for_date_range( args.server, job_name, dates, max_parallel=args.max_parallel, ignore_errors=(not args.fail_on_error), ), ) finally: loop.close() print_backfill_runs_table(backfill_runs) yield all(br.run_state in BackfillRun.SUCCESS_STATES for br in backfill_runs) def tron_version(args): local_version = __version__ print(f"Tron client version: {local_version}") response = client.request(urljoin(args.server, "/api/status")) if response.error: print(f"Error: {response.content}") yield server_version = response.content.get("version", "unknown") print(f"Tron server version: {server_version}") if server_version != local_version: print("Warning: client and server versions should match") yield yield True COMMANDS: dict[str, Callable[[argparse.Namespace], Generator[bool, None, None]]] = defaultdict( lambda: control_objects, publish=event_publish, discard=event_discard, backfill=backfill, move=move, retry=retry, version=tron_version, ) def main(): """run tronctl""" args = parse_cli() cmd_utils.load_config(args) # NOTE: we do this after load_configs() since load_config() may set some logging defaults that we want to override desired_level = cmd_utils.setup_logging(args) logging.getLogger().setLevel(desired_level) cmd = COMMANDS[args.command] try: for ret in cmd(args): if not ret: sys.exit(ExitCode.fail) except RequestError as err: print( f"Error connecting to the tron server ({args.server}): {err}", file=sys.stderr, ) sys.exit(ExitCode.fail) if __name__ == "__main__": main() ================================================ FILE: bin/tronctl_tabcomplete.sh ================================================ if [[ -n ${ZSH_VERSION-} ]]; then autoload -U +X bashcompinit && bashcompinit fi # This magic eval enables tab-completion for tron commands # http://argcomplete.readthedocs.io/en/latest/index.html#synopsis eval "$(/opt/venvs/tron/bin/register-python-argcomplete tronctl)" ================================================ FILE: bin/trond ================================================ #!/usr/bin/env python """ Start the Tron server daemon.""" import argparse import faulthandler import logging import os import time import traceback import pkg_resources import tron from tron import trondaemon from tron.commands import cmd_utils from tron.config import manager log = logging.getLogger(__name__) DEFAULT_CONF = "default_config.yaml" DEFAULT_CONF_PATH = "config/" DEFAULT_WORKING_DIR = "/var/lib/tron/" DEFAULT_LOCKFILE = "tron.lock" DEFAULT_LOCKPATH = "/var/run/" + DEFAULT_LOCKFILE def parse_cli(): parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", version=f"{parser.prog} {tron.__version__}", ) parser.add_argument( "-w", "--working-dir", default=DEFAULT_WORKING_DIR, help="Working directory for the Tron daemon, default %(default)s", ) parser.add_argument( "-c", "--config-path", default=DEFAULT_CONF_PATH, help="File path to the Tron configuration file", ) parser.add_argument( "--nodaemon", action="store_true", default=False, help="[DEPRECATED] Disable daemonizing, default %(default)s", ) parser.add_argument( # for backwards compatibility "--pid-file", help="[DEPRECATED] File path to pid file. Use --lock-file instead.", ) parser.add_argument( "--lock-file", help="File path to lock file, defaults to %s if working directory " "is default. Otherwise defaults to /%s" % (DEFAULT_LOCKPATH, DEFAULT_LOCKFILE), ) logging_group = parser.add_argument_group("logging", "") logging_group.add_argument( "--log-conf", "-l", help="File path to a custom logging.conf", ) logging_group.add_argument( "-v", "--verbose", action="count", default=0, help="Verbose logging. Repeat for more verbosity.", ) logging_group.add_argument( "--debug", action="store_true", help="Debug mode, extra error reporting, no daemonizing", ) api_group = parser.add_argument_group("Web Service API", "") api_group.add_argument( "--port", "-P", dest="listen_port", type=int, help="TCP port number to listen on, default %(default)s", default=cmd_utils.DEFAULT_PORT, ) api_group.add_argument( "--host", "-H", dest="listen_host", help="Hostname to listen on, default %(default)s", default=cmd_utils.DEFAULT_HOST, ) requirement = pkg_resources.Requirement.parse("tron") api_group.add_argument( "--web-path", default=pkg_resources.resource_filename( requirement, "tronweb", ), help="Path to static web resources, default %(default)s.", ) args = parser.parse_args() args.working_dir = os.path.abspath(args.working_dir) if args.log_conf: args.log_conf = os.path.join(args.working_dir, args.log_conf) if not os.path.exists(args.log_conf): parser.error("Logging config file not found: %s" % args.log_conf) if not args.lock_file: if args.pid_file: # for backwards compatibility args.lock_file = args.pid_file elif args.working_dir == DEFAULT_WORKING_DIR: args.lock_file = DEFAULT_LOCKPATH else: args.lock_file = DEFAULT_LOCKFILE args.lock_file = os.path.join(args.working_dir, args.lock_file) args.config_path = os.path.join( args.working_dir, args.config_path, ) return args def create_default_config(config_path): """Create a default empty configuration for first time installs""" default = pkg_resources.resource_string(tron.__name__, DEFAULT_CONF) manager.create_new_config(config_path, default) def setup_environment(args): """Setup the working directory and config environment.""" if not os.path.exists(args.working_dir): os.makedirs(args.working_dir) if not os.path.isdir(args.working_dir) or not os.access(args.working_dir, os.R_OK | os.W_OK | os.X_OK): msg = "Error, can't access working directory %s" % args.working_dir raise SystemExit(msg) # Attempt to create a default config if config is missing if not os.path.exists(args.config_path): try: create_default_config(args.config_path) except OSError as e: msg = "Error creating default configuration at %s: %s" log.debug(traceback.format_exc()) raise SystemExit(msg % (args.config_path, e)) if not os.access(args.config_path, os.R_OK | os.W_OK): msg = "Error opening configuration %s: Missing Permissions" raise SystemExit(msg % args.config_path) def main(): args = parse_cli() boot_time = time.time() setup_environment(args) trond = trondaemon.TronDaemon(args) trond.run(boot_time) if __name__ == "__main__": # print tracebacks on signals/faults # NOTE: you likely want to read https://docs.python.org/3/library/faulthandler.html # as these tracebacks will look slightly different faulthandler.enable() try: main() # this is a little weird, but every now and then we're seeing a mysterious tron exit # that doesn't seem to correspond with anything else - let's catch BaseException # (and therefore SystemExit) in case anything is calling sys.exit() since there's no # traceback when we see this except ( BaseException, # technically, we really only need to catch BaseException - but let's be extra-paranoid Exception, ): traceback.print_exc() raise ================================================ FILE: bin/tronfig ================================================ #!/usr/bin/env python import logging import os import shutil import sys import tempfile import traceback from tron.commands import cmd_utils from tron.commands.client import Client from tron.config import config_parse from tron.config import ConfigError from tron.config import manager from tron.config import schema log = logging.getLogger("tronfig") def parse_cli(): parser = cmd_utils.build_option_parser() parser.add_argument( "-p", "--print", action="store_true", dest="print_config", help="Print config to stdout, rather than uploading", ) parser.add_argument( "-C", "--check", action="store_true", dest="check", help="Upload and check configuration, don't apply, " "useful when you want to verify if tron daemon " "will accept your configuration.", ) parser.add_argument( "-d", "--delete", action="store_true", help="Delete the configuration for this namespace", ) parser.add_argument( "-V", "--validate", action="store_true", dest="validate", help="Only validate configuration, don't upload, " "useful for verifying config locally. If namespace " "is not specified, it will be derived from file " "name, if any.", ) parser.add_argument( "-D", "--validate-dir", action="store_true", dest="validate_dir", help="Full validation of a folder, don't upload, " "same as -V but checks for more edge-cases", ) parser.add_argument( "-n", "--namespace", action="store", help="Alternate namespace to use", ) parser.add_argument( "-m", "--master-config", action="store", dest="master_config", help="Source of master configuration file", ) parser.add_argument("source") return parser.parse_args() def upload_config(client, config_name, contents, config_hash, check=False): response = client.config( config_name, config_data=contents, config_hash=config_hash, check=check, ) if "error" in response: log.error(response["error"]) return False print("Configuration uploaded successfully", file=sys.stderr) return True def validate(config_name, config_content, master_content=None): try: config_data = manager.from_string(config_content) master_data = ( manager.from_string( master_content, ) if master_content else None ) config_parse.validate_fragment( name=config_name, fragment=config_data, master_config=master_data, ) except ConfigError as e: return str(e) def delete_config(client, config_name): if config_name == schema.MASTER_NAMESPACE: log.error( "Deleting MASTER namespace is not allowed. Name must be specified.", ) return response = input( f"This will delete the configuration for the {config_name} namespace. Proceed? (y/n): ", ) if response[:1].lower() != "y": return config_hash = client.config(config_name)["hash"] if upload_config(client, config_name, "", config_hash): return raise SystemExit("tronfig deletion failed") def validate_dir(path): try: manifest_dir = tempfile.mkdtemp() manifest = manager.ManifestFile(manifest_dir) manifest.create() for fname in os.listdir(path): name, ext = os.path.splitext(fname) if ext == ".yaml": namespace = name manifest.add(namespace, os.path.join(path, fname)) config_manager = manager.ConfigManager(path, manifest) config_manager.load() except ConfigError as e: traceback.print_exc() return str(e) finally: if manifest_dir: shutil.rmtree(manifest_dir) def get_config_input(namespace, source): if source == "-": source_io = sys.stdin if not namespace: namespace = schema.MASTER_NAMESPACE else: source_io = open(source) if not namespace: name, _ = os.path.splitext(os.path.basename(source)) namespace = name content = source_io.read() return namespace, content if __name__ == "__main__": args = parse_cli() cmd_utils.setup_logging(args) cmd_utils.load_config(args) if args.validate or args.validate_dir: if args.validate: name, content = get_config_input(args.namespace, args.source) master_content = None if args.master_config: _, master_content = get_config_input( schema.MASTER_NAMESPACE, args.master_config, ) result = validate( config_name=name, config_content=content, master_content=master_content, ) elif args.validate_dir: result = validate_dir(args.source) if not result: print("OK") sys.exit(0) else: print(result) sys.exit(1) client = Client(args.server) if args.print_config: content = client.config(args.source)["config"] if type(content) is not bytes: content = content.encode("utf8") os.write(sys.stdout.fileno(), content) elif args.delete: delete_config(client, args.source) else: namespace, content = get_config_input(args.namespace, args.source) config_hash = client.config(namespace)["hash"] result = validate(namespace, content) if result: print(result) sys.exit(1) if upload_config( client, namespace, content, config_hash, check=args.check, ): sys.exit(0) print("Uploading failed") sys.exit(1) ================================================ FILE: bin/tronrepl ================================================ #!/usr/bin/env python """ Start the Tron server daemon.""" import argparse import logging import os import traceback import IPython import pkg_resources import tron.mcp from tron import trondaemon from tron.commands import cmd_utils from tron.config import manager log = logging.getLogger(__name__) DEFAULT_CONF = "default_config.yaml" DEFAULT_CONF_PATH = "config/" DEFAULT_WORKING_DIR = "/var/lib/tron/" DEFAULT_LOCKFILE = "tron-repl.lock" DEFAULT_LOCKPATH = "/var/run/" + DEFAULT_LOCKFILE def parse_cli(): parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", version=f"{parser.prog} {tron.__version__}", ) parser.add_argument( "-w", "--working-dir", default=DEFAULT_WORKING_DIR, help="Working directory for the Tron daemon, default %(default)s", ) parser.add_argument( "-c", "--config-path", default=DEFAULT_CONF_PATH, help="File path to the Tron configuration file", ) parser.add_argument( "--nodaemon", action="store_true", default=False, help="[DEPRECATED] Disable daemonizing, default %(default)s", ) parser.add_argument( # for backwards compatibility "--pid-file", help="[DEPRECATED] File path to pid file. Use --lock-file instead.", ) parser.add_argument( "--lock-file", help="File path to lock file, defaults to %s if working directory " "is default. Otherwise defaults to /%s" % (DEFAULT_LOCKPATH, DEFAULT_LOCKFILE), ) logging_group = parser.add_argument_group("logging", "") logging_group.add_argument( "--log-conf", "-l", help="File path to a custom logging.conf", ) logging_group.add_argument( "-v", "--verbose", action="count", default=0, help="Verbose logging. Repeat for more verbosity.", ) logging_group.add_argument( "--debug", action="store_true", help="Debug mode, extra error reporting, no daemonizing", ) api_group = parser.add_argument_group("Web Service API", "") api_group.add_argument( "--port", "-P", dest="listen_port", type=int, help="TCP port number to listen on, default %(default)s", default=cmd_utils.DEFAULT_PORT, ) api_group.add_argument( "--host", "-H", dest="listen_host", help="Hostname to listen on, default %(default)s", default=cmd_utils.DEFAULT_HOST, ) requirement = pkg_resources.Requirement.parse("tron") api_group.add_argument( "--web-path", default=pkg_resources.resource_filename( requirement, "tronweb", ), help="Path to static web resources, default %(default)s.", ) args = parser.parse_args() args.working_dir = os.path.abspath(args.working_dir) if args.log_conf: args.log_conf = os.path.join(args.working_dir, args.log_conf) if not os.path.exists(args.log_conf): parser.error("Logging config file not found: %s" % args.log_conf) if not args.lock_file: if args.pid_file: # for backwards compatibility args.lock_file = args.pid_file elif args.working_dir == DEFAULT_WORKING_DIR: args.lock_file = DEFAULT_LOCKPATH else: args.lock_file = DEFAULT_LOCKFILE args.lock_file = os.path.join(args.working_dir, args.lock_file) args.config_path = os.path.join( args.working_dir, args.config_path, ) return args def create_default_config(config_path): """Create a default empty configuration for first time installs""" default = pkg_resources.resource_string(tron.__name__, DEFAULT_CONF) manager.create_new_config(config_path, default) def setup_environment(args): """Setup the working directory and config environment.""" if not os.path.exists(args.working_dir): os.makedirs(args.working_dir) if not os.path.isdir(args.working_dir) or not os.access(args.working_dir, os.R_OK | os.W_OK | os.X_OK): msg = "Error, can't access working directory %s" % args.working_dir raise SystemExit(msg) # Attempt to create a default config if config is missing if not os.path.exists(args.config_path): try: create_default_config(args.config_path) except OSError as e: msg = "Error creating default configuration at %s: %s" log.debug(traceback.format_exc()) raise SystemExit(msg % (args.config_path, e)) if not os.access(args.config_path, os.R_OK | os.W_OK): msg = "Error opening configuration %s: Missing Permissions" raise SystemExit(msg % args.config_path) def main(): args = parse_cli() setup_environment(args) trond = trondaemon.TronDaemon(args) # noqa: F841 trond.mcp = tron.mcp.MasterControlProgram( trond.options.working_dir, trond.options.config_path, ) trond.mcp._load_config() # trond.mcp.restore_state(trond.mcp.config.load().get_master().action_runner) # mcp = trond.mcp # noqa: F841 # store = mcp.state_watcher.state_manager._impl # noqa: F841 print("") print("+---------------------+") print("| Tron REPL |") print("| Available locals: |") print("| - trond |") print("+---------------------+") print("") IPython.embed() if __name__ == "__main__": main() ================================================ FILE: bin/tronview ================================================ #!/usr/bin/env python import os import sys import argcomplete from tron.commands import cmd_utils from tron.commands import display from tron.commands.client import Client from tron.commands.client import get_object_type_from_identifier from tron.commands.client import RequestError from tron.commands.client import TronObjectType from tron.commands.cmd_utils import ExitCode from tron.commands.cmd_utils import suggest_possibilities from tron.commands.cmd_utils import tron_jobs_completer def parse_cli(): parser = cmd_utils.build_option_parser() parser.add_argument( "--numshown", "-n", type=int, dest="num_displays", help="Max number of jobs/job-runs shown", default=10, ) parser.add_argument( "--color", "-c", action="store_true", dest="display_color", help="Display in color", default=None, ) parser.add_argument( "--nocolor", action="store_false", dest="display_color", help="Display without color", default=None, ) parser.add_argument( "--stdout", "-o", action="count", dest="stdout", help="Solely displays stdout", default=0, ) parser.add_argument( "--stderr", "-e", action="count", dest="stderr", help="Solely displays stderr", default=0, ) parser.add_argument( "--events", "-E", action="store_true", dest="events", help="Display stored events", default=0, ) parser.add_argument( "name", nargs="?", help="job name | job run id | action id", ).completer = cmd_utils.tron_jobs_completer argcomplete.autocomplete(parser) args = parser.parse_args() return args def console_height(): if not sys.stdout.isatty(): return 40 return int(os.popen("stty size", "r").read().split()[0]) def view_all(args, client): """Retrieve jobs and display them.""" return display.DisplayJobs().format( client.jobs( include_job_runs=False, include_action_runs=False, include_action_graph=False, include_node_pool=False, ), ) def view_job(args, job_id, client): """Retrieve details of the specified job and display""" job_content = client.job(job_id.url, count=args.num_displays) return display.format_job_details(job_content) def view_job_run(args, job_run_id, client): actions = client.job_runs(job_run_id.url) display_action = display.DisplayActionRuns() return display_action.format(actions) def view_action_run(args, act_run_id, client): content = client.action_runs( act_run_id.url, num_lines=args.num_displays, ) return display.format_action_run_details(content) obj_type_to_view_map = { TronObjectType.job: view_job, TronObjectType.job_run: view_job_run, TronObjectType.action_run: view_action_run, } def get_view_output(name, args, client): url_index = client.index() try: tron_id = get_object_type_from_identifier(url_index, name) except ValueError as e: possibilities = list(tron_jobs_completer(prefix="", client=client)) suggestions = suggest_possibilities( word=name, possibilities=possibilities, ) raise SystemExit(f"Error: {e}{suggestions}") if tron_id.type not in obj_type_to_view_map: return try: return obj_type_to_view_map[tron_id.type](args, tron_id, client) except RequestError as e: raise SystemExit(f"Error: {e}") def main(): """run tronview""" args = parse_cli() cmd_utils.setup_logging(args) cmd_utils.load_config(args) display.Color.toggle(args.display_color) client = Client(args.server) try: if args.events: response = client.request("/api/events") error = response.get("error") if not error: for evt in response.get("response", ["* no recorded events *"]): print(evt) sys.exit(ExitCode.success) if not args.name: output = view_all(args, client) else: output = get_view_output(args.name, args, client) if not output: print("Unrecognized identifier: %s" % args.name, file=sys.stderr) sys.exit(ExitCode.fail) if sys.stdout.isatty() and len(output.split("\n")) > console_height(): display.view_with_less(output, args.display_color) else: print(output) except RequestError as err: print( f"Error connecting to the tron server ({args.server}): {err}", file=sys.stderr, ) sys.exit(ExitCode.fail) if __name__ == "__main__": main() ================================================ FILE: bin/tronview_tabcomplete.sh ================================================ if [[ -n ${ZSH_VERSION-} ]]; then autoload -U +X bashcompinit && bashcompinit fi # This magic eval enables tab-completion for tron commands # http://argcomplete.readthedocs.io/en/latest/index.html#synopsis eval "$(/opt/venvs/tron/bin/register-python-argcomplete tronview)" ================================================ FILE: contrib/migration_script.py ================================================ #!/usr/bin/env python """ This script is for migrating jobs to another namespace """ import argparse import subprocess import time from urllib.parse import urljoin from urllib.parse import urlparse from tron import yaml from tron.commands import client class bcolors: HEADER = "\033[95m" OKBLUE = "\033[94m" OKGREEN = "\033[92m" WARNING = "\033[93m" FAIL = "\033[91m" ENDC = "\033[0m" BOLD = "\033[1m" UNDERLINE = "\033[4m" def parse_args(): parser = argparse.ArgumentParser( description="Migrate jobs to new namespace", ) parser.add_argument( "--server", required=True, help="specify the location of tron master", ) parser.add_argument( "--old-ns", required=True, help="Old namespace", ) parser.add_argument( "--new-ns", required=True, help="New namespace", ) parser.add_argument( "source", help="source file to get list of jobs", ) parser.add_argument( "--job", help="Specify a single job to migrate", ) args = parser.parse_args() return args def check_job_if_running(jobs_status, job_name): for job_status in jobs_status: if job_status["name"] == job_name: status = job_status["status"] if status == "running": print(bcolors.FAIL + f"job {job_name} is still running, can not migrate" + bcolors.ENDC) return False elif status == "disabled": print(bcolors.WARNING + f"job {job_name} is disabled, need to cancel it manually later" + bcolors.ENDC) return True else: print(bcolors.OKGREEN + f"job {job_name} is not running, can migrate" + bcolors.ENDC) return True print(bcolors.FAIL + f"Can not find the job {job_name}" + bcolors.ENDC) return False def command_jobs(command, jobs, args, ns=None): """This function run tronctl command for the jobs command: the tronctl command it will run jobs: a list of jobs args: the args for this script ns: the namespace to use as the prefix for each job, if None, the scrip would use args.old_ns instead """ data = {"command": command} command_flag = True for job in jobs: if ns is not None: job_name = ns + "." + job["name"] else: job_name = args.old_ns + "." + job["name"] if command == "move": data = { "command": command, "old_name": args.old_ns + "." + job["name"], "new_name": args.new_ns + "." + job["name"], } uri = urljoin(args.server, "api/jobs") job_name = args.new_ns + "." + job["name"] else: data = {"command": command} uri = urljoin(args.server, "api/jobs/" + job_name) response = client.request(uri, data=data) if response.error: print(bcolors.FAIL + f"Failed to {command} {job_name}" + bcolors.ENDC) command_flag = False else: print(bcolors.OKGREEN + f"Succeed to {command} {job_name}" + bcolors.ENDC) return command_flag def ssh_command(hostname, command): print(bcolors.BOLD + f"Executing the command: ssh -A {hostname} {command}" + bcolors.ENDC) ssh = subprocess.Popen( ["ssh", "-A", hostname, command], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) exitcode = ssh.wait() result = ssh.stdout.readlines() error = ssh.stderr.readlines() if exitcode != 0: print(bcolors.FAIL + f"Execute command {command} failed: {error}" + bcolors.ENDC) exit(exitcode) return result def main(): args = parse_args() filename = args.source hostname = urlparse(args.server).hostname if filename.endswith(".yaml"): tron_client = client.Client(args.server) jobs_status = tron_client.jobs() is_migration_safe = True with open(filename) as f: jobs = yaml.load(f)["jobs"] job_names = [job["name"] for job in jobs] if args.job is not None: # only want to migrate specific job # Overwrite existing jobs since only migrating one job jobs = [job for job in jobs if job["name"] == args.job] if not jobs: raise ValueError(f"Invalid job specified. Options were {job_names}") job_name_with_ns = args.old_ns + "." + args.job is_migration_safe = is_migration_safe & check_job_if_running(jobs_status, job_name_with_ns) else: # Migrate all jobs in namespace for job_name in job_names: job_name_with_ns = args.old_ns + "." + job_name is_migration_safe = is_migration_safe & check_job_if_running(jobs_status, job_name_with_ns) if is_migration_safe is True: print(bcolors.OKBLUE + "Jobs are not running." + bcolors.ENDC) else: print(bcolors.WARNING + "Some jobs are still running, abort this migration," + bcolors.ENDC) return # try stop cron ssh_command(hostname, "sudo service cron stop") # wait unitil yelpsoa-configs branch is merged res = input("Merge and push yelpsoa-configs branch. Ready to continue? [y/n]") if res == "y": # wait for 10 seconds after pushing the branch time.sleep(30) # rsyn yelpsoa-configs command = "sudo rsync -a --delay-updates --contimeout=10 --timeout=10 --chmod=Du+rwx,go+rx --port=8731 --delete yelpsoa-slave.local.yelpcorp.com::yelpsoa-configs /nail/etc/services" ssh_command(hostname, command) # migrate jobs to new namespace command_jobs("move", jobs, args) # update new namespace ssh_command(hostname, "sudo paasta_setup_tron_namespace " + args.new_ns) # update old namespace if only one job is moving if args.job: ssh_command(hostname, "sudo paasta_setup_tron_namespace " + args.old_ns) # clean up namespace ssh_command(hostname, "sudo paasta_cleanup_tron_namespaces") # start cron ssh_command(hostname, "sudo service cron start") return if __name__ == "__main__": main() ================================================ FILE: contrib/mock_patch_checker.py ================================================ #!/usr/bin/env python3.10 import ast import sys class MockChecker(ast.NodeVisitor): def __init__(self): self.errors = 0 self.init_module_imports() def init_module_imports(self): self.imported_patch = False self.imported_mock = False def check_files(self, files): for file in files: self.check_file(file) def check_file(self, filename): self.current_filename = filename try: with open(filename) as fd: try: file_ast = ast.parse(fd.read()) except SyntaxError as error: print("SyntaxError on file %s:%d" % (filename, error.lineno)) return except OSError: print("Error opening filename: %s" % filename) return self.init_module_imports() self.visit(file_ast) def _call_uses_patch(self, node): try: return node.func.id == "patch" except AttributeError: return False def _call_uses_mock_patch(self, node): try: return node.func.value.id == "mock" and node.func.attr == "patch" except AttributeError: return False def visit_Import(self, node): if [name for name in node.names if "mock" == name.name]: self.imported_mock = True def visit_ImportFrom(self, node): if node.module == "mock" and (name for name in node.names if "patch" == name.name): self.imported_patch = True def visit_Call(self, node): try: if (self.imported_patch and self._call_uses_patch(node)) or ( self.imported_mock and self._call_uses_mock_patch(node) ): if not any([keyword for keyword in node.keywords if keyword.arg == "autospec"]): print("%s:%d: Found a mock without an autospec!" % (self.current_filename, node.lineno)) self.errors += 1 except AttributeError: pass self.generic_visit(node) def main(filenames): checker = MockChecker() checker.check_files(filenames) if checker.errors == 0: sys.exit(0) else: print("You probably meant to specify 'autospec=True' in these tests.") print("If you really don't want to, specify 'autospec=None'") sys.exit(1) if __name__ == "__main__": main(sys.argv[1:]) ================================================ FILE: contrib/namespace_cleanup.sh ================================================ #/bin/bash ecosystem="stagef" read -p "Are you at tron-$ecosystem (y/n)?" RES echo if [ $RES = "y" ]; then #load namespace from _manifest.yaml for namespace in $(cat /nail/tron/config/_manifest.yaml | uq | jq -r 'keys[]') do file=$(cat /nail/tron/config/_manifest.yaml | uq | jq -r .\"$namespace\") filename=$(basename $file) if [ -f "/nail/etc/services/tron/$ecosystem/$filename" ]; then echo "$namespace is up to date" elif [ $namespace == "MASTER" ]; then echo "It is MASTER namepsace" else num_job=$(cat /nail/tron/config/$filename | uq | jq -r ".jobs | length") echo "========= $filename =========" cat /nail/tron/config/$filename echo "=============================" if [ $num_job == 0 ]; then echo "$namespace is left behind, deleting the namespace" tronfig -d $namespace else echo "Can't remove the namespace since it is not empty." fi fi done else echo "Please change the ecosystem variable in this script or execute this script at tron-$ecosystem" fi ================================================ FILE: contrib/patch-config-loggers.diff ================================================ --- a/debian/tron/opt/venvs/tron/lib/python3.10/site-packages/kubernetes/client/configuration.py +++ b/debian/tron/opt/venvs/tron/lib/python3.10/site-packages/kubernetes/client/configuration.py @@ -71,11 +71,11 @@ """ _default = None - def __init__(self, host="http://localhost", api_key=None, api_key_prefix=None, username=None, password=None, discard_unknown_keys=False, + is_logger_used=False, ): """Constructor """ @@ -106,26 +106,28 @@ """Password for HTTP basic authentication """ self.discard_unknown_keys = discard_unknown_keys + self.is_logger_used = is_logger_used self.logger = {} - """Logging Settings - """ - self.logger["package_logger"] = logging.getLogger("client") - self.logger["urllib3_logger"] = logging.getLogger("urllib3") - self.logger_format = '%(asctime)s %(levelname)s %(message)s' - """Log format - """ - self.logger_stream_handler = None - """Log stream handler - """ - self.logger_file_handler = None - """Log file handler - """ - self.logger_file = None - """Debug file location - """ - self.debug = False - """Debug switch - """ + if self.is_logger_used: + """Logging Settings + """ + self.logger["package_logger"] = logging.getLogger("client") + self.logger["urllib3_logger"] = logging.getLogger("urllib3") + self.logger_format = '%(asctime)s %(levelname)s %(message)s' + """Log format + """ + self.logger_stream_handler = None + """Log stream handler + """ + self.logger_file_handler = None + """Log file handler + """ + self.logger_file = None + """Debug file location + """ + self.debug = False + """Debug switch + """ self.verify_ssl = True """SSL/TLS verification @@ -178,11 +180,12 @@ for k, v in self.__dict__.items(): if k not in ('logger', 'logger_file_handler'): setattr(result, k, copy.deepcopy(v, memo)) - # shallow copy of loggers - result.logger = copy.copy(self.logger) - # use setters to configure loggers - result.logger_file = self.logger_file - result.debug = self.debug + if self.is_logger_used: + # shallow copy of loggers + result.logger = copy.copy(self.logger) + # use setters to configure loggers + result.logger_file = self.logger_file + result.debug = self.debug return result @classmethod ================================================ FILE: contrib/sync-from-yelp-prod.sh ================================================ #!/bin/bash rsync --exclude=.stderr --exclude=.stdout -aPv tron-prod:/nail/tron/* example-cluster/ git checkout example-cluster/logging.conf echo "" echo "Now Run:" echo "" echo " tox -e example-cluster" echo " ./example-cluster/start.sh" ================================================ FILE: contrib/sync_namespaces_jobs.py ================================================ #!/usr/bin/env python """ This script is for load testing of Tron Historically, Tronview and Tronweb were (are) slow. To better understand the performance bottleneck of Tron, we could use this script to generate the fake namespaces and jobs as many as we want to perform load testing. Ticket TRON-70 tracks the progress of speeding up Tronview and Tronweb. """ import argparse import os from tron import yaml def parse_args(): parser = argparse.ArgumentParser( description="Creating namespaces and jobs configuration for load testing", ) parser.add_argument( "--multiple", type=int, default=1, help="multiple workload of namespaces and jobs from source directory", ) parser.add_argument( "--src", default="/nail/etc/services/tron/prod", help="Directory to get Tron configuration files", ) parser.add_argument( "--dest", default="/tmp/tron-servdir", help="Directory to put Tron configuration files for load testing", ) args = parser.parse_args() return args def main(): args = parse_args() for filename in os.listdir(args.src): print(f"filename = {filename}") filepath = os.path.join(args.src, filename) if os.path.isfile(filepath) and filepath.endswith(".yaml"): with open(filepath) as f: config = yaml.load(f) if filename == "MASTER.yaml": for key in list(config): if key != "jobs": del config[key] jobs = config.get("jobs", []) if jobs is not None: for job in jobs: job["node"] = "localhost" if "monitoring" in job: del job["monitoring"] for action in job.get("actions", []): action["command"] = "sleep 10s" if "node" in action: action["node"] = "localhost" for i in range(args.multiple): out_filepath = os.path.join( args.dest, "load_testing_" + str(i) + "-" + filename, ) with open(out_filepath, "w") as outf: yaml.dump(config, outf, default_flow_style=False) if __name__ == "__main__": main() ================================================ FILE: debian/changelog ================================================ tron (3.10.0) jammy; urgency=medium * 3.10.0 tagged with 'make release' Commit: Merge pull request #1096 from Yelp/u/kkasp/unlearn-how-to- read-pickles U/kkasp/unlearn how to read pickles -- Kevin Kaspari Mon, 16 Mar 2026 09:03:19 -0700 tron (3.9.3) jammy; urgency=medium * 3.9.3 tagged with 'make release' Commit: Merge pull request #1098 from jhereth/u/jhereth/MLCOMPUTE- 6270/add-attempt-number-label Add k8s label for attempt number -- Kevin Kaspari Mon, 16 Mar 2026 07:40:42 -0700 tron (3.9.2) jammy; urgency=medium * 3.9.2 tagged with 'make release' Commit: Merge pull request #1090 from Yelp/u/kkasp/TRON-2452- compress-json Compress json -- Kevin Kaspari Mon, 09 Feb 2026 13:24:43 -0800 tron (3.9.1) jammy; urgency=medium * 3.9.1 tagged with 'make release' Commit: Merge pull request #1084 from Yelp/jfong/TRON-2546-fix-spot- terminatioons-not-ooms TRON-2546: Fix spot termination identification in current k8s version -- Jen Patague Thu, 08 Jan 2026 09:58:32 -0800 tron (3.9.0) jammy; urgency=medium * 3.9.0 tagged with 'make release' Commit: Upgrade Tron to Python 3.10 - TRON-2435 (#1071) * Upgrade Tron to Python 3.10 * Upgrade zope.interface to a compatible python 3.10 version * Moved moto to requirements dev instead and removed mock from minimal * lowering object_size for more room * Update classifier and path in itest -- Eman Elsabban Thu, 18 Dec 2025 06:24:50 -0800 tron (3.8.9) jammy; urgency=medium * 3.8.9 tagged with 'make release' Commit: Assume we"ll only see yaml files in logreader.py (#1077) If someone is using a .yml file - they"ve already gone horribly off- track :p -- Luis Perez Mon, 24 Nov 2025 13:42:34 -0800 tron (3.8.8) jammy; urgency=medium * 3.8.8 tagged with 'make release' Commit: Try to warn folks about incorrect `tronctl publish` usage (#1076) It"s absurdly easy to use this incorrectly (especially if you"ve been woken up by a page ;p) - so let"s try to warn folks if they"re potentially using `tronctl publish` incorrectly afaik, the usual mistakes here are to use a job/action id (detectable by the run number) OR to use a too long/short trigger id (either from copy- pasting too much or too little) I"ve opted to not error out when this happens since I"m somewhat sure that it"s possible to manually construct triggers that don"t match this format and depend on those triggers in soaconfigs. If anyone with more time wants to vibe-code something to verify this (and maybe even add some validation to the tronfig schema ;)), then we can always error out immediately -- Luis Perez Thu, 20 Nov 2025 12:01:45 -0800 tron (3.8.7) jammy; urgency=medium * 3.8.7 tagged with 'make release' Commit: Lightly refactor skip-and-publish (and add some more output) (#1075) The current output is pretty minimal and doesn"t help us (or users) debug particularly efficiently whenever something isn"t working as expected. Output now looks something like: ``` Skipping compute-infra-test-service.always_fail.5.fail... ActionRun: compute- infra-test-service.always_fail.5.fail now in state skipped Successfully skipped compute-infra-test-service.always_fail.5.fail. Fetching triggers to publish for compute-infra-test- service.always_fail.5.fail... Triggers to publish: * compute- infra-test-service.always_fail.fail.shortdate.2025-11-15 Publishing trigger compute-infra-test-service.always_fail.fail.shortdate.2025- 11-15... OK ``` I"ve lightly refactored this since I don"t really know why I inlined all this logic in 2021 and it was bothering me. -- Luis Perez Wed, 12 Nov 2025 06:46:31 -0800 tron (3.8.6) jammy; urgency=medium * 3.8.6 tagged with 'make release' Commit: Add another disk eviction substring (#1074) We"re also seeing messages that look like: `Pod ephemeral local storage usage exceeds the total limit of containers 1Gi.` which don"t trigger the existing check. This adds "ephemeral local storage" as another trigger without removing the existing one. I"m not quite sure how k8s decides which message to use without reading through the k8s code further, but it appears that both cases are possible: https://github.com/kubernetes/kubernetes/blob/350481c4747da5c2ad4f24 e71e7edaabbd4cfe2e/pkg/kubelet/eviction/helpers.go#L53-L57 -- Luis Perez Mon, 03 Nov 2025 09:56:04 -0800 tron (3.8.5) jammy; urgency=medium * 3.8.5 tagged with 'make release' Commit: TRON-2480: Fix tron systemd unit to retry indefinitely & add minor delay between restarts (#1070) We saw in https://yelp.slack.com/archives/CA53K7S68/p1755111528654579 and https://yelp.slack.com/archives/CA53K7S68/p1752625385958589?thread_t s=1752624525.373959&cid=CA53K7S68 my change from #1066 to add the `ExecStartPre` , combined with the default `RestartSec=0` and `StartLimitBurst=5` meant that tron in a cluster that required some time to perform the shutdown process would still see `trond` still running and immediately run through the 5 allowed attempts, then fail to restart unless retriggered by a timer or socket: > Note that units which are configured for Restart=, and which reach the start limit are not attempted to be restarted anymore; however, they may still be restarted manually or from a timer or socket at a later point, after the interval has passed. This PR now disables the restart limits entirely by setting both `StartLimitBurst` and `StartLimitIntervalSec` to 0, as per the docs: > interval is a time span with the default unit of seconds, but other units may be specified, see [systemd.time(7)](https://www.freedesktop.org/software/systemd/man/l atest/systemd.time.html#). The special value "infinity" can be used to limit the total number of start attempts, even if they happen at large time intervals. Defaults to DefaultStartLimitIntervalSec= in manager configuration file, and may be set to 0 to disable any kind of rate limiting. (Note this indicates only setting `StartLimitIntervalSec=0` is necessary to disable the rate limiting, but chatgpt convinced me it makes it clearer to the reader if both are set to 0, happy to undo this if we think we"ll be confused later) These settings seem to work on tron-infrastage after modifying it to cause an artificial 60s delay when shutting down, as we can see restarts are delayed 10s between the next attempt, and eventually tron is able to start cleanly on its own with no user intervention: https://fluffy.yelpcorp.com/i/qnk5kPRNXsMPRbpLv32KvKDLFk1gl72p.html -- Luis Perez Mon, 18 Aug 2025 09:22:27 -0700 tron (3.8.4) jammy; urgency=medium * 3.8.4 tagged with 'make release' Commit: Default sliders to show 50 items (#1067) Defaulting to 10 doesn"t actually save any time since we"ve fetched all the data anyway. Let"s instead default to 50 since that should should be the most common run_limit This will make pages longer, but it will be less confusing to folks when they get alerts for failed/unknown/etc jobs that are past the visible runs with the previous slider configuration. -- Luis Perez Wed, 23 Jul 2025 14:10:18 -0700 tron (3.8.3) jammy; urgency=medium * 3.8.3 tagged with 'make release' Commit: Merge pull request #1066 from Yelp/jfong/TRON-2340-systemd- sigterm-handling TRON-2340: Update tron unit file to ensure proper sigterm handling -- Jen Patague Mon, 14 Jul 2025 13:51:48 -0700 tron (3.8.2) jammy; urgency=medium * 3.8.2 tagged with 'make release' Commit: Released 3.8.1 via make release -- Jen Patague Mon, 14 Jul 2025 12:42:31 -0700 tron (3.8.1) jammy; urgency=medium * 3.8.1 tagged with 'make release' Commit: Enable disallow_untyped_decorators mypy option (#1064) For once, a small mypy PR :p There was just a single untyped decorator and I went ahead and wrote some inline comments "cause I don"t think I"ll remember how to read this after this gets merged :deepfried- loljpg: -- Luis Perez Wed, 02 Jul 2025 10:14:27 -0700 tron (3.8.0) jammy; urgency=medium * 3.8.0 tagged with 'make release' Commit: Allow transitioning to success/fail from any terminal state (#1057) This is helpful when folks have handled failures outside of Tron (or have done something outside of Tron that nominally means that the Tron status is now incorrect) OR to work-around any weirdness with Tron states (e.g., something funky happened in k8s and Tron never saw the event) NOTE: this PR changes a bunch of extra files - but that"s just due to mypy following more code paths after I typed the ActionRunController `__init__` so that I could go- to-def in my editor :p Additionally, kkasp and I have occasionally been runing into an state where `check-requirements` says a mismatched version of `cryptography` is installed - I finally got around to debugging this and realized this stems from our very hacky extra_requirements_yelp.txt shenanigans: pyopenssl has lower/upper- bounds on cryptography but, due to the ordering we install things in, `check-requirements` might run before we"ve installed pyopenssl and downgraded cryptography (and pass) or after (and fail) The real changes are confined to `tron/core/action.py::ActionRun::STATE_MACHINE` (and a now-deleted ActionRun tests that are no longer valid) -- Luis Perez Mon, 30 Jun 2025 10:16:53 -0700 tron (3.7.3) jammy; urgency=medium * 3.7.3 tagged with 'make release' Commit: more gracefully handle errors during backfill monitoring (#1062) -- Matteo Piano Thu, 26 Jun 2025 05:54:02 -0700 tron (3.7.2) jammy; urgency=medium * 3.7.2 tagged with 'make release' Commit: Merge pull request #1056 from Yelp/u/kkasp/rest-in-peace- tronweb2 Delete tronweb2 -- Kevin Kaspari Wed, 25 Jun 2025 10:25:34 -0700 tron (3.7.1) jammy; urgency=medium * 3.7.1 tagged with 'make release' Commit: Merge pull request #1055 from Yelp/u/kkasp/TRON-2007-tron- disk-evictions Add disk eviction error code and check logic -- Kevin Kaspari Fri, 20 Jun 2025 10:11:01 -0700 tron (3.7.0) jammy; urgency=medium * 3.7.0 tagged with 'make release' Commit: Save idempotent to json - TRON-2154 (#1051) * Save idempotent to json * set idempotent to false if it doesnt exist * address review -- Eman Elsabban Wed, 18 Jun 2025 13:09:02 -0700 tron (3.6.3) jammy; urgency=medium * 3.6.3 tagged with 'make release' Commit: ignore cached okta token in backfills (#1054) -- Matteo Piano Mon, 16 Jun 2025 01:38:02 -0700 tron (3.6.2) jammy; urgency=medium * 3.6.2 tagged with 'make release' Commit: cache auth tokens before starting backfill jobs (#1053) -- Matteo Piano Fri, 13 Jun 2025 01:23:13 -0700 tron (3.6.1) jammy; urgency=medium * 3.6.1 tagged with 'make release' Commit: return auth error that clients can parse (#1052) -- Matteo Piano Mon, 09 Jun 2025 05:24:25 -0700 tron (3.6.0) jammy; urgency=medium * 3.6.0 tagged with 'make release' Commit: Merge pull request #1046 from Yelp/u/kkasp/TRON-2391- aggregate-metrics U/kkasp/tron 2391 aggregate metrics -- Kevin Kaspari Wed, 04 Jun 2025 10:43:44 -0700 tron (3.5.2) jammy; urgency=medium * 3.5.2 tagged with 'make release' Commit: use oidc token method from vault-tools (#1050) -- Matteo Piano Wed, 28 May 2025 07:36:00 -0700 tron (3.5.1) jammy; urgency=medium * 3.5.1 tagged with 'make release' Commit: fix some annoying warnings (#1049) -- Matteo Piano Tue, 27 May 2025 10:22:44 -0700 tron (3.5.0) jammy; urgency=medium * 3.5.0 tagged with 'make release' Commit: add support for api auth via vault (#1048) add support for api auth via vault -- Matteo Piano Tue, 27 May 2025 08:47:48 -0700 tron (3.4.10) jammy; urgency=medium * 3.4.10 tagged with 'make release' Commit: Merge pull request #1045 from Yelp/u/kkasp/TRON-2414-fix- broken-config-parse U/kkasp/tron 2414 fix broken config parse -- Kevin Kaspari Thu, 08 May 2025 09:51:28 -0700 tron (3.4.9) jammy; urgency=medium * 3.4.9 tagged with 'make release' Commit: Merge pull request #1043 from Yelp/u/kkasp/TRON-2414-set- extra-safe-transaction-limit Drop under the limit a bit more for maximum safety -- Kevin Kaspari Thu, 01 May 2025 10:54:54 -0700 tron (3.4.8) jammy; urgency=medium * 3.4.8 tagged with 'make release' Commit: fix build_url_request when method is None (#1041) -- Luis Perez Mon, 14 Apr 2025 10:52:51 -0700 tron (3.4.7) jammy; urgency=medium * 3.4.7 tagged with 'make release' Commit: Merge pull request #1042 from Yelp/u/kkasp/DAR-2637-lower- max-transact-items Reduce transact size to avoid cap -- Kevin Kaspari Thu, 10 Apr 2025 08:24:26 -0700 tron (3.4.6) jammy; urgency=medium * 3.4.6 tagged with 'make release' Commit: Merge pull request #1039 from Yelp/u/kkasp/KKASP-0001- download-more-time Upgrade moment and get the latest tz info -- Kevin Kaspari Mon, 07 Apr 2025 11:10:20 -0700 tron (3.4.5) jammy; urgency=medium * 3.4.5 tagged with 'make release' Commit: Bump task_processing from 1.3.4 to 1.3.5 (#1040) I made a silly mistake and didn"t increase the attempt counter -- Luis Perez Thu, 03 Apr 2025 15:37:22 -0700 tron (3.4.4) jammy; urgency=medium * 3.4.4 tagged with 'make release' Commit: Update task_processing to 1.3.4 for watch backoff (#1038) This includes https://github.com/Yelp/task_processing/pull/225, which should add some backoff to watch restarts to avoid slamming the apiserver -- Luis Perez Thu, 03 Apr 2025 14:01:31 -0700 tron (3.4.3) jammy; urgency=medium * 3.4.3 tagged with 'make release' Commit: Bump kubernetes clientlib to the latest supported version (#1037) See https://github.com/kubernetes-client/python?tab=readme- ov-file#compatibility -- Luis Perez Thu, 03 Apr 2025 10:46:44 -0700 tron (3.4.2) jammy; urgency=medium * 3.4.2 tagged with 'make release' Commit: Merge pull request #1036 from Yelp/u/kkasp/TRON-2396-lower- min-zoom-for-big-ol-charts Halve minZoom for some of our big charts -- Kevin Kaspari Thu, 27 Mar 2025 10:42:14 -0700 tron (3.4.1) jammy; urgency=medium * 3.4.1 tagged with 'make release' Commit: Bump task_processing to v1.3.3 for Watch restart fix (#1035) This brings in https://github.com/Yelp/task_processing/pull/223, which should ensure that we don"t get stuck in a funky restart loop if k8s tells us our watch"s resourceVersion is too old -- Luis Perez Wed, 26 Mar 2025 11:46:03 -0700 tron (3.4.0) jammy; urgency=medium * 3.4.0 tagged with 'make release' Commit: Merge pull request #1032 from Yelp/u/kkasp/TRON-2370-do-ya- like-dags U/kkasp/tron 2370 do ya like dags -- Kevin Kaspari Fri, 21 Mar 2025 13:17:02 -0700 tron (3.3.2) jammy; urgency=medium * 3.3.2 tagged with 'make release' Commit: Merge pull request #1034 from Yelp/u/mpiano/SEC- 19862_fix_service_param fix service name extraction logic -- Kevin Kaspari Fri, 21 Mar 2025 11:28:33 -0700 tron (3.3.1) jammy; urgency=medium * 3.3.1 tagged with 'make release' Commit: Merge pull request #1033 from Yelp/u/mpiano/SEC-19862_fix fix auth middleware integration -- Kevin Kaspari Mon, 17 Mar 2025 12:47:36 -0700 tron (3.3.0) jammy; urgency=medium * 3.3.0 tagged with 'make release' Commit: Merge pull request #1005 from Yelp/u/mpiano/SEC-19555 auth support for Tron APIs -- Kevin Kaspari Thu, 13 Mar 2025 11:30:42 -0700 tron (3.2.12) jammy; urgency=medium * 3.2.12 tagged with 'make release' Commit: Merge pull request #1030 from Yelp/jfong/TRON-1797-tronweb- path TRON-1797: Put tronweb in a python-version-agnostic location -- Jen Patague Mon, 03 Mar 2025 12:15:00 -0800 tron (3.2.11) jammy; urgency=medium * 3.2.11 tagged with 'make release' Commit: Try to load config_name_mapping from disk just once (#1013) There"s enough files (and enough YAML in these files) that the IO/YAML parsing takes a significant amount of time. While it"s nice to always load from the source-of-truth (i.e., the files on-disk) - it"s not worth paying the performance penalty (especially at the scale we"re seeing internally) for a negligible benefit. Locally (with a large test config), this results in a ~5x improvement in timings. More concretely, my test configs were taking ~30ish seconds *without* this diff and ~6ish seconds *with* this diff. -- Luis Perez Mon, 24 Feb 2025 09:08:41 -0800 tron (3.2.10) jammy; urgency=medium * 3.2.10 tagged with 'make release' Commit: Merge pull request #1028 from Yelp/jfong/TRON-2333 TRON- 2333: Bump task_processing for pod truncation fix -- Jen Patague Fri, 21 Feb 2025 11:08:55 -0800 tron (3.2.9) jammy; urgency=medium * 3.2.9 tagged with 'make release' Commit: Merge pull request #1023 from Yelp/u/kkasp/TRON-2342- exponential-backoff-dynamo-get Add dynamodb retry config for throttling and other errors. Add exponential backoff and jitter for unprocessed keys. Fix edge case where we succesfully process keys on our last attempt but still fail -- Kevin Kaspari Wed, 12 Feb 2025 13:03:36 -0800 tron (3.2.8) jammy; urgency=medium * 3.2.8 tagged with 'make release' Commit: Merge pull request #1025 from Yelp/u/kkasp/DAR-2558-fix- overstep Fix loop boundary on getting partitions -- Kevin Kaspari Thu, 30 Jan 2025 11:59:51 -0800 tron (3.2.7) jammy; urgency=medium * 3.2.7 tagged with 'make release' Commit: Merge pull request #1027 from Yelp/u/emanelsabban/TRON-2354 Handling timezone aware jobs -- Eman Elsabban Fri, 24 Jan 2025 12:38:21 -0800 tron (3.2.6) jammy; urgency=medium * 3.2.6 tagged with 'make release' Commit: Add default behaviour for jobs that dont have some keys and handle the non-existent of some json_vals (#1017) * Add default behaviour for jobs that dont have some keys and handle the non- existent of some json_vals * Remove get from command * cap add and drop should have values * removing the try/except since it was added in a separate pr * maybe not all gets are necessary * Adding comments -- Eman Elsabban Fri, 17 Jan 2025 11:47:48 -0800 tron (3.2.5) jammy; urgency=medium * 3.2.5 tagged with 'make release' Commit: Merge pull request #1026 from Yelp/u/kkasp/fix-timezones Use localize instead of replace when writing tz info -- Kevin Kaspari Fri, 17 Jan 2025 11:00:55 -0800 tron (3.2.4) jammy; urgency=medium * 3.2.4 tagged with 'make release' Commit: Merge pull request #1024 from Yelp/u/kkasp/empty-val Add check for empty val. Merge json correctly -- Kevin Kaspari Thu, 16 Jan 2025 08:49:56 -0800 tron (3.2.3) jammy; urgency=medium * 3.2.3 tagged with 'make release' Commit: Merge pull request #1016 from Yelp/u/kkasp/TRON-2239-migrate- pickle-only-state-to-JSON U/kkasp/tron 2239 migrate pickle only state to json -- Kevin Kaspari Tue, 14 Jan 2025 12:05:21 -0800 tron (3.2.2) jammy; urgency=medium * 3.2.2 tagged with 'make release' Commit: Merge pull request #1014 from Yelp/u/kkasp/fix-tooltip-lol- its-bootstrap-2.3.1 Disable the tooltip animation that is causing a strange problem where you can"t get a tooltip when entering from the top of an object -- Kevin Kaspari Mon, 13 Jan 2025 06:59:56 -0800 tron (3.2.1) jammy; urgency=medium * 3.2.1 tagged with 'make release' Commit: Merge pull request #1022 from Yelp/u/kkasp/pass-json-back-to- save-queue Pass json_val back to the save queue on failure -- Kevin Kaspari Fri, 10 Jan 2025 07:27:17 -0800 tron (3.2.0) jammy; urgency=medium * 3.2.0 tagged with 'make release' Commit: Merge pull request #1018 from Yelp/u/kkasp/update-delete- item Update delete_item logic to handle json partitions -- Kevin Kaspari Thu, 09 Jan 2025 12:29:29 -0800 tron (3.1.2) jammy; urgency=medium * 3.1.2 tagged with 'make release' Commit: Bump task_processing to v1.3.1 for capability fix (#1020) Just like Yelp/paasta#3972 and Yelp/paasta#3973, we need to ensure that there are no duplicates between cap_add and cap_drop - otherwise, the cap_drop entry will "win" and the duplicate capability will not be added. -- Luis Perez Thu, 09 Jan 2025 09:21:33 -0800 tron (3.1.1) jammy; urgency=medium * 3.1.1 tagged with 'make release' Commit: Merge pull request #1019 from Yelp/u/emanelsabban/capture- json_val Capturing exception when json_val doesnt exist -- Eman Elsabban Wed, 08 Jan 2025 13:04:07 -0800 tron (3.1.0) jammy; urgency=medium * 3.1.0 tagged with 'make release' Commit: Add read_json to be a part of master.yaml config (#1015) * Add read_json to be a part of master.yaml config * Fix the mypy error & other failed tests * Addressing reviews * Add reas_json to restore in shelve function -- Eman Elsabban Thu, 02 Jan 2025 12:51:55 -0800 tron (3.0.0) jammy; urgency=medium * 3.0.0 tagged with 'make release' Commit: [TRON-2238] Reading from JSON to restore jobs" state instead of pickles (#1010) * Reading from Json to restore jobs" state instead of pickles * Fixing some bugs through testing * Deleting some comments * mocking get_config_watcher * Try mocking one more time * Toggling off read_json since we want to merge that way * Removing some comments * Addressed reviews * Toggling read_json back off * Addressing more reviews * one more review plz * includes reviews except tz -- Eman Elsabban Mon, 23 Dec 2024 07:56:01 -0800 tron (2.8.1) jammy; urgency=medium * 2.8.1 tagged with 'make release' Commit: Merge pull request #1004 from Yelp/u/kkasp/TRON-2237- timedeltas-cant-json Use total_seconds for timedeltas. Log job and jobrun names/runnums on serialization -- Kevin Kaspari Thu, 31 Oct 2024 09:11:35 -0700 tron (2.8.0) jammy; urgency=medium * 2.8.0 tagged with 'make release' Commit: Merge pull request #997 from Yelp/u/kkasp/TRON-2237-write- json-state Write JSON state -- Kevin Kaspari Tue, 29 Oct 2024 14:14:38 -0700 tron (2.7.0) jammy; urgency=medium * 2.7.0 tagged with 'make release' Commit: Merge pull request #1001 from Yelp/u/cuza/getting_namespace_logs_from_yelpsoa Fixing Tron logs for jobs using other services images -- Kevin Kaspari Tue, 29 Oct 2024 13:31:16 -0700 tron (2.6.0) jammy; urgency=medium * 2.6.0 tagged with 'make release' Commit: Use logreader rathern than vector-reader (#1002) This CLI is being renamed, so let"s account for that -- Luis Perez Thu, 24 Oct 2024 09:34:05 -0700 tron (2.5.3) jammy; urgency=medium * 2.5.3 tagged with 'make release' Commit: Merge pull request #1000 from Yelp/jfong/TRON-2208- nonretryable-to-unknown TRON-2208: Update non_retryable_exit_code behavior to treat as UNKNOWN -- Jen Patague Mon, 30 Sep 2024 14:12:37 -0700 tron (2.5.2) jammy; urgency=medium * 2.5.2 tagged with 'make release' Commit: Use vector-reader cli if new logging pipeline is enabled (#999) Thankfully, this was a pretty easy change - we really only needed to parametrize the command name being used and the location selector is now a lot simpler as it"ll always be the superregion -- Luis Perez Mon, 30 Sep 2024 09:32:22 -0700 tron (2.5.1) jammy; urgency=medium * 2.5.1 tagged with 'make release' Commit: Merge pull request #998 from Yelp/jfong/TRON-2277-fix- nonretryable-exit-codes TRON-2277: Pass along non_retryable_exit_codes to KubernetesCluster objects -- Jen Patague Thu, 26 Sep 2024 10:21:53 -0700 tron (2.5.0) jammy; urgency=medium * 2.5.0 tagged with 'make release' Commit: Attempt to batch config loading for deployments (#996) Right now we make at most 2N calls to the Tron API during config deployments: N to get the current configs and at most N if all services have changes. To start, I"d like to reduce this to N by allowing GET /api/config to return all the configs so that the only requests needed are POSTs for changed configs. Depending on how this goes, we can look into batching up the POSTs so that we can also do that in a single request. In terms of speed, it looks like loading all the configs from pnw-prod (on my devbox) with this new behavior takes ~3s - which isn"t great, but there"s a decent bit of file IO going on here :( -- Luis Perez Tue, 17 Sep 2024 08:25:15 -0700 tron (2.4.2) jammy; urgency=medium * 2.4.2 tagged with 'make release' Commit: Merge pull request #995 from Yelp/u/cuza/making-non- retryable-exit-code-accept-negative-numbers Fix negative value check for non-retryable exit codes in Kubernetes configuration -- Dave Cuza Mon, 09 Sep 2024 12:23:49 -0700 tron (2.4.1) jammy; urgency=medium * 2.4.1 tagged with 'make release' Commit: Merge pull request #994 from Yelp/StefanoChiodino-patch-1 Update tronweb.less -- Luis Perez Tue, 03 Sep 2024 11:24:45 -0700 tron (2.4.0) jammy; urgency=medium * 2.4.0 tagged with 'make release' Commit: TRON-2208: Add toggle in tron config to disable retries on LOST k8s jobs (#988) Given that "LOST" means Tron has lost track of a pod it already thought it had started for a job, attempting to retry/start a replacement can be dangerous for non-idempotent jobs. In the current state, these will consume retries, but with some of our EKS migration methods, LOST tasks are more likely. Therefore, we should have a way to temporarily pause retries on these. ## Related Issues - TRON-2208: Add toggle in Tron config to disable retries on LOST k8s jobs -- Luis Perez Wed, 28 Aug 2024 09:19:14 -0700 tron (2.3.0) jammy; urgency=medium * 2.3.0 tagged with 'make release' Commit: Merge pull request #989 from Yelp/jfong/TRON-2195-old- kubeconfig-paths TRON-2195: Support watcher_kubeconfig_paths -- Jen Patague Thu, 11 Jul 2024 14:25:42 -0700 tron (2.2.7) jammy; urgency=medium * 2.2.7 tagged with 'make release' Commit: Only show disable warning on tronctl disable (#990) I"m not sure I was thinking here since this ended up unconditionally printing the disable warning - but we can chalk this up to an intentional PR campaign to warn folks and only show the warnings on tronctl disable from now on :p -- Luis Perez Thu, 11 Jul 2024 11:49:18 -0700 tron (2.2.6) jammy; urgency=medium * 2.2.6 tagged with 'make release' Commit: Update scribereader tests (#983) -- Yaroslav Liakhovskyi Fri, 28 Jun 2024 01:41:29 -0700 tron (2.2.5) jammy; urgency=medium * 2.2.5 tagged with 'make release' Commit: Update yelp_clog and use datetime range for S3 logs (#979) -- Yaroslav Liakhovskyi Wed, 26 Jun 2024 08:27:10 -0700 tron (2.2.4) jammy; urgency=medium * 2.2.4 tagged with 'make release' Commit: Merge pull request #981 from Yelp/revert-980- u/jfong/revert_to_2.1.1 Revert "Reverts all changes back through 2.1.1 and retains urgent unprocessedkeys fix" -- Eman Elsabban Thu, 20 Jun 2024 14:52:09 -0700 tron (2.2.3) jammy; urgency=medium * 2.2.3 tagged with 'make release' Commit: Reverts all changes back through 2.1.1 and retains urgent unprocessedkeys fix (#980) * Revert "pass around projected SA configs properly" This reverts commit 12b1bf27e7b9e9f3fd0963f9d073d2552a58115f. * Revert "Merge remote- tracking branch "origin/u/mpiano/SEC-18955"" This reverts commit ea6376d72ffdd269e11cb9338d4f0c656bcd6f66, reversing changes made to 4038f1e173aeff932b6e060fc2690f7aa502a85d. * Revert "Use S3LogsReader with superregion and UTC timezone (#972)" This reverts commit af73799363549a082eea11f0f98d5c7f4810abe4. -- Eman Elsabban Thu, 20 Jun 2024 10:07:42 -0700 tron (2.2.2) jammy; urgency=medium * 2.2.2 tagged with 'make release' Commit: Merge pull request #977 from Yelp/u/jfong/tron_quick_fix Fix UnprocessedKeys bug when restoring from dynamodb -- Jen Patague Mon, 17 Jun 2024 17:50:59 -0700 tron (2.2.1) jammy; urgency=medium * 2.2.1 tagged with 'make release' Commit: Merge pull request #976 from Yelp/u/mpiano/SEC-18955_fix pass around projected SA configs properly -- Jen Patague Mon, 17 Jun 2024 14:52:24 -0700 tron (2.2.0) jammy; urgency=medium * 2.2.0 tagged with 'make release' Commit: Merge remote-tracking branch "origin/u/mpiano/SEC-18955" -- Matteo Piano Tue, 11 Jun 2024 02:27:25 -0700 tron (2.1.2) jammy; urgency=medium * 2.1.2 tagged with 'make release' Commit: Use S3LogsReader with superregion and UTC timezone (#972) -- Yaroslav Liakhovskyi Wed, 05 Jun 2024 02:17:04 -0700 tron (2.1.1) jammy; urgency=medium * 2.1.1 tagged with 'make release' Commit: Handling exceptions thrown from threads span by ThreadPoolExecutor - TRON-2202 (#969) * Handling exceptions thrown from threads span by ThreadPoolExecutor * Addressing reviews * Addressing more reviews * Removed Exit code -- Eman Elsabban Tue, 04 Jun 2024 11:24:00 -0700 tron (2.1.0) jammy; urgency=medium * 2.1.0 tagged with 'make release' Commit: Revert the Mesos code deletions (#970) This turned out to be unsafe to to our use of pickles as a serialization format. * Revert "Use the latest task_proc (#966)" This reverts commit 01003a980854bc25ed2764c880e1fb69db296fb1. * Revert "Delete remaining Mesos code (#961)" This reverts commit 1f71d0fa406e530ac943f1fcaf312224015f392c. * Revert "Delete Mesos related exit codes and docker files - TRON-2187 (#959)" This reverts commit 33ad2a1657aeea61f899380dada42825e137083b. * Revert "Delete Mesos logging config (#962)" This reverts commit 640362424f1b1b6bb1c959a04b0843ca1e846c10. * Revert "Merge pull request #953 from Yelp/u/emanelsabban/TRON-2183" This reverts commit 10353457221d11f2a587c665bae05b16f3cba447, reversing changes made to e4114088fefaf6a3d3f5be27e5caf41d7c1c9973. * Revert "Deleting Mesos code from the Master Control Program" This reverts commit 18f48ee1db23e1e7f91e0e9e7846eb345d171534. -- Luis Perez Mon, 03 Jun 2024 13:33:05 -0700 tron (2.0.0) jammy; urgency=medium * 2.0.0 tagged with 'make release' Commit: Use the latest task_proc (#966) There"s no real changes here other than dropping all the Mesos-related code from task_proc -- Luis Perez Fri, 17 May 2024 13:49:55 -0700 tron (1.32.5) jammy; urgency=medium * 1.32.5 tagged with 'make release' Commit: Merge pull request #965 from Yelp/u/emanelsabban/fix-make- release Remove /docs/source/generated from gitignore file -- Eman Elsabban Fri, 17 May 2024 09:10:54 -0700 tron (1.32.4) jammy; urgency=medium * 1.32.4 tagged with 'make release' Commit: Delete Mesos related exit codes and docker files - TRON-2187 (#959) * Delete Mesos related exit codes and docker files * Adding a try/except for validating extra keys * Testing deleted extra keys -- Eman Elsabban Fri, 17 May 2024 07:51:23 -0700 tron (1.32.3) jammy; urgency=medium * 1.32.3 tagged with 'make release' Commit: Automate make release in Tron (#960) -- Jon Lee Mon, 13 May 2024 15:36:34 -0700 tron (1.32.2) jammy; urgency=medium * 1.32.2 tagged with 'make release' Commit: Merge pull request #953 from Yelp/u/emanelsabban/TRON-2183 Delete Mesos code from statemanager - TRON-2183 -- Eman Elsabban Mon, 13 May 2024 12:00:06 -0700 tron (1.32.1) jammy; urgency=medium * 1.32.1 tagged with 'make release' Commit: Merge pull request #952 from Yelp/u/emanelsabban/TRON-2182 Deleting Mesos code from the Master Control Program - TRON-2182 -- Eman Elsabban Mon, 13 May 2024 11:43:05 -0700 tron (1.32.0) jammy; urgency=medium * 1.32.0 tagged with 'make release' Commit: Update task_proc to better handle killing tasks (#951) We've seen that task_proc/k8s will sometimes not correctly send events for pods that we try to kill ourselves (either because the pods are already gone or because the event is somehow missing data), so this task_proc version will send synthetic events when we call kill() to ensure that tron is in the correct state :) Co-authored-by: Jen Patague -- Luis Perez Thu, 09 May 2024 09:16:24 -0700 tron (1.31.0) jammy; urgency=medium * 1.31.0 tagged with 'make release' Commit: Parallelizing execution of restoring in Tron - TRON-2161 (#950) * Parallelizing execution of restoring in Tron * Addressing reviews and fixing unit tests * Adding sorting for runs state otherwise runs will be out of order * Addressing more reviews and adding exception for exceeding max_attempts * Address typing stuff and timers * Fixing the typing error * fixing incomplete sentence -- Eman Elsabban Wed, 01 May 2024 12:54:10 -0700 tron (1.30.0) jammy; urgency=medium * 1.30.0 tagged with 'make release' Commit: Adding yelp_clog S3LogsReader (#949) * Upgrade yelp scribereader deps * Enable S3LogsReader for action run logs * Upgrade boto requirements * Upgrade mypy and update type ignores * Pin more internal requirements -- Yaroslav Liakhovskyi Mon, 29 Apr 2024 04:57:37 -0700 tron (1.29.5) jammy; urgency=medium * 1.29.5 tagged with 'make release' Commit: Merge pull request #940 from Yelp/jfong/TRON-1850-starting- jobs-stuck TRON-1850: Include 'starting' pods in check for stuck jobs -- Jen Patague Tue, 09 Apr 2024 11:37:15 -0700 tron (1.29.4) jammy; urgency=medium * 1.29.4 tagged with 'make release' Commit: Adding logs to Tron to indicate start and scheduling times - TRON-2152 (#948) * Adding logs to Tron to indicate start and scheduling times * Addressed reviews on the PR * Address reviews 2.0 * Deleting the start_schedule_jobs flag * Making boot_time a required param * bring back time.time in duration -- Eman Elsabban Mon, 08 Apr 2024 11:15:48 -0700 tron (1.29.3) jammy; urgency=medium * 1.29.3 tagged with 'make release' Commit: Add a top-level exception handler (#947) As the comment says: let's see if adding a top-level handler (that then re-raises) and catching BaseException will give us more info as to what's sometimes causing Tron to exit. I've also added the usual base exception (Exception) just to be extra-safe (h/t to krall for the idea) -- Luis Perez Wed, 27 Mar 2024 15:00:00 -0700 tron (1.29.2) jammy; urgency=medium * 1.29.2 tagged with 'make release' Commit: Merge pull request #946 from Yelp/u/emanelsabban/edit- messages Editing some of the messages from previous prs -- Eman Elsabban Wed, 20 Mar 2024 08:33:45 -0700 tron (1.29.1) jammy; urgency=medium * 1.29.1 tagged with 'make release' Commit: Merge pull request #942 from Yelp/u/kkasp/TRON-1970-escape- html-tronweb Use Underscores HTML escaped interpolation in template for log and command outputs -- Kevin Kaspari Fri, 08 Mar 2024 08:03:14 -0800 tron (1.29.0) jammy; urgency=medium * 1.29.0 tagged with 'make release' Commit: Merge pull request #944 from Yelp/u/emanelsabban/TRON-2124- expose-prom-metrics Adding prometheus endpoint in tron - TRON-2124 -- Eman Elsabban Tue, 05 Mar 2024 12:55:14 -0800 tron (1.28.5) jammy; urgency=medium * 1.28.5 tagged with 'make release' Commit: Stop keeping old copies of eventbus files (#941) We've never restored from these 'backups', and as long as we ensure that the current file points to a fully written file, it's fine to only keep at most 2 files around: the actual current file and a temporary 'new' file. To ensure that we're not pointing at a half-written file, we continue using the age-old pattern for atomic file updates: write to another file and swap a symlink once the write is complete -- Luis Perez Tue, 20 Feb 2024 11:55:45 -0800 tron (1.28.4) jammy; urgency=medium * 1.28.4 tagged with 'make release' Commit: Merge pull request #939 from Yelp/u/wilmerrafael/COMPINFRA- 3601_adding_tron_run_number_label Adding tron run id to pod labels for k8s -- Wilmer Bandres Thu, 15 Feb 2024 06:43:40 -0800 tron (1.28.3) jammy; urgency=medium * 1.28.3 tagged with 'make release' Commit: Merge pull request #938 from Yelp/u/kkasp/TRON-2112-lock- start Add lock to tron start to mitigate the risk of running duplicate jobs… -- Kevin Kaspari Thu, 08 Feb 2024 11:24:39 -0800 tron (1.28.2) jammy; urgency=medium * 1.28.2 tagged with 'make release' Commit: Remove unnecessary mocks (#937) This is what I get for trusting GHA rather than also running the tests internally :) These mocks are no longer required and are actually causing test failures internally. -- Luis Perez Thu, 01 Feb 2024 09:10:42 -0800 tron (1.28.1) jammy; urgency=medium * 1.28.1 tagged with 'make release' Commit: Downpin yelp-clog (#936) We had bumped this a couple major versions since we were also bumping scribereader - but we reverted the scribereader bump before merging the jammy/py38 branch and forgot to also revert the yelp-clog bump :) -- Luis Perez Thu, 01 Feb 2024 08:51:40 -0800 tron (1.28.0) jammy; urgency=medium * 1.28.0 tagged with 'make release' Commit: Upgrading Tron to py3.8 + patching it with the fix (#934) * Monkeypatch SimpleQueue back to PySimpleQueue We have a hunch that this is what is causing our pod event loop to have wildly delayed items * Revert 'Revert python/jammy upgrades (#907)' This reverts commit 483da5c0fb258b01b8e47912ad034d43554ada7d. * new formatting * Bump pyyaml * Rm sad test that is not relevant for validation * added stuff to run tron locally * matching the python version with whats currently running in infrastage * adding also changelog * This commit includes some requriements for clog and try/except block for handle_events * This commit adds the patch fix * Revert 'matching the python version with whats currently running in infrastage' This reverts commit 1f81a6d4805d742a7a7a28b1d7d8eef39522a896. * Revert 'adding also changelog' This reverts commit 951fcc8fc31114bde340bad4b82ced0529e03b65. * precommit fixes the patch * Fixing mypy issues and tests failing * removing return and adding comment for handling defer being none * addressing wording * fix InvariantException back to Exception --------- Co-authored- by: Luis Perez Co-authored-by: Vincent Thibault -- Eman Elsabban Wed, 31 Jan 2024 09:25:13 -0800 tron (1.27.5) jammy; urgency=medium * 1.27.5 tagged with 'make release' Commit: Allow Tron to use more than 10k file descriptors (#923) There appears to be an fd leak somewhere in tron, we're not in danger of hitting any host-level limits - so let's update the unit file to have a bigger limit in the meantime -- Luis Perez Wed, 11 Oct 2023 12:11:22 -0700 tron (1.27.4) jammy; urgency=medium * 1.27.4 tagged with 'make release' Commit: Revert 'Link tronweb_url ' (#928) Reverts #911 as it did not actually result in a clickable link in Slack -- Luis Perez Tue, 10 Oct 2023 12:26:07 -0700 tron (1.27.3) jammy; urgency=medium * 1.27.3 tagged with 'make release' Commit: Catch all exceptions in k8s submit_command() (#926) It's entirely possible that creating a task_processing task (and/or submitting one) can result in an exception. At the moment, this results in the affected ActionRun getting stuck in the Starting state - but this is a lie and means that the normal monitoring/alerting on failed runs does not kick in. -- Luis Perez Mon, 21 Aug 2023 12:27:26 -0700 tron (1.27.2) jammy; urgency=medium * 1.27.2 tagged with 'make release' Commit: Merge pull request #924 from Yelp/u/vit/fix- configsecretvolume Fix item assignment issue with ConfigSecretVolume -- Vincent Thibault Mon, 14 Aug 2023 13:36:37 -0700 tron (1.27.1) jammy; urgency=medium * 1.27.1 tagged with 'make release' Commit: Update to the latest task_proc (#915) This version pulls in a fix that stops pods from being launched with a null request. This has been running fine in infrastage where I've verified that Pods are indeed being launched with the correct metadata I've verified this as well with unit tests in task_proc :) -- Luis Perez Mon, 10 Jul 2023 08:28:15 -0700 tron (1.27.0) jammy; urgency=medium * 1.27.0 tagged with 'make release' Commit: Remove tronweb2 code (#914) This isn't currently being used and is complicating the internal build process -- Luis Perez Tue, 13 Jun 2023 08:49:02 -0700 tron (1.26.0) jammy; urgency=medium * 1.26.0 tagged with 'make release' Commit: Merge pull request #909 from Yelp/u/vit/tron-secret-volume TRON-1636: Pass secret_volumes to taskproc from kubernetes -- Vincent Thibault Mon, 12 Jun 2023 09:03:39 -0700 tron (1.25.1) jammy; urgency=low * 1.25.1 tagged with 'make release' Commit: Merge pull request #911 from Yelp/tchen/link-tronweb-url Link tronweb_url -- Tianle Chen Mon, 05 Jun 2023 09:21:09 -0700 tron (1.25.0) jammy; urgency=medium * 1.25.0 tagged with 'make release' Commit: COMPINFRA-2565: enforce an upper limit on backfill concurrency (#906) Problem ----- The Kubernetes control plane can be overwhelmed if a high number of concurrent backfills are executed. We currently set a relatively high default of and enforce no upper limit at all. Solution ----- * Reduce the default from to to reduce the baseline pressure of Tron backfill jobs. * Introduce and enforce a hard limit of concurrent backfills per tronctl invocation In a future PR, we might want to track the total count of running backfills per user in Zookeeper but this change should already safeguard against overwhelming Tron with backfills in the meantime. Signed-off-by: Max Falk -- Luis Perez Thu, 18 May 2023 09:06:24 -0700 tron (1.24.5) jammy; urgency=medium * 1.24.5 tagged with 'make release' Commit: Revert python/jammy upgrades (#907) * Revert 'Upgrades requirements and adds caching on get loggers (#900)' This reverts commit 0ac2b69d648bb3b915a80637a4f3051e06a16296. * Revert 'Adds Python 3.8; build for Jammy (#896)' This reverts commit f7679b9a281d026b4d956c5735875eded5602310. -- Luis Perez Wed, 17 May 2023 08:44:16 -0700 tron (1.24.4) jammy; urgency=medium * 1.24.4 tagged with 'make release' Commit: Updates date arithmetic to handle whitespace (#903) -- Jon Lee Thu, 09 Mar 2023 14:23:44 -0800 tron (1.24.3) jammy; urgency=medium * 1.24.3 tagged with 'make release' Commit: Upgrades kubernetes version to support 1.21 (#902) -- Jon Lee Thu, 09 Mar 2023 12:15:55 -0800 tron (1.24.2) jammy; urgency=medium * 1.24.2 tagged with 'make release' Commit: Ignores CryptographyDeprecationWarning (#901) * Ignores CryptographyDeprecationWarning -- Jon Lee Thu, 09 Mar 2023 11:46:44 -0800 tron (1.24.1) jammy; urgency=medium * 1.24.1 tagged with 'make release' Commit: Upgrades requirements and adds caching on get loggers (#900) * updates axe-core * Upgrade twisted * Stop locking on getting loggers * Adds better description and commit suggestions * Upgrades Werkzeug dependency * Upgrades future dependency * Upgrades ipython dependency * Upgrades cryptography dependency * Upgrades setuptools dependency * Upgrades certifi dependency * Upgrades urllib3 dependency * Upgrades rsa dependency * Upgrades Pygments dependency * Upgrades jinja2 for docs dependency * Upgrades docs dependencies * Update tron/trondaemon.py Co-authored- by: Luis Pérez --------- Co-authored-by: Luis Pérez -- Jon Lee Wed, 08 Mar 2023 10:13:38 -0800 tron (1.24.0) jammy; urgency=medium * 1.24.0 tagged with 'make release' Commit: Fixes make release for jammy (#899) -- Jon Lee Fri, 03 Mar 2023 05:39:51 -0800 tron (1.23.10) bionic; urgency=medium * 1.23.10 tagged with 'make release' Commit: Merge pull request #890 from Yelp/u/cuza/logging_tron_exit_status Adding ExecStopPost to systemd unit file to log tron exit status -- root Tue, 31 Jan 2023 18:27:19 +0000 tron (1.23.9) bionic; urgency=medium * 1.23.9 tagged with 'make release' Commit: Merge pull request #887 from Yelp/u/jfong/TRON-1723 TRON- 1723: Add user attribution into user agent when invoking tronctl commands -- root Wed, 07 Dec 2022 20:29:42 +0000 tron (1.23.8) bionic; urgency=medium * 1.23.8 tagged with 'make release' Commit: TRON-1825: Add additional failure handling of specific errors (#886) Adds additional failure handling of specific errors Adds handling of spot interruptions and k8s scaling down Captures real exit codes from failures. -- root Mon, 05 Dec 2022 15:43:23 +0000 tron (1.23.7) bionic; urgency=medium * 1.23.7 tagged with 'make release' Commit: Use correct dict casing for k8s metadata (#885) The data we'd been dumping into our logging streams was using different key names than what task_proc was actually sending k8s: we were logging the python-ified names, but task_proc was sending tron the literal k8s payloads (i.e., keynames being camelCased). We didn't notice this 'cause until the initial attempt at working around this weird k8s bug we weren't actually logging what tron was seeing and our test data in the unit tests was based on what we'd been seeing in our logging streams -- root Wed, 23 Nov 2022 19:44:24 +0000 tron (1.23.6) bionic; urgency=medium * 1.23.6 tagged with 'make release' Commit: Detect abnormal successful exits in k8s (#884) this is kinda wild: we're seeing that a kubelet will sometimes fail to start a container (usually due to what appear to be race conditions like those mentioned in https://github.com/kubernetes/kubernetes/issues/100047#issuecomment- 797624208 and then decide that these Pods should be phase=Succeeded with an exit code of 0 - even though the container never actually started. So far, we've noticed that when this happens, the and fields will be - thus we'll check for at least one of these conditions to detect an abnormal exit and actually 'fail' the affected action -- root Tue, 22 Nov 2022 19:28:02 +0000 tron (1.23.5) bionic; urgency=medium * 1.23.5 tagged with 'make release' Commit: Merge pull request #882 from Yelp/DAR-1739-revert-rookout Revert 'This adds rookout to tron (TRON-1764) (#875)' -- root Wed, 26 Oct 2022 00:24:33 +0000 tron (1.23.4) bionic; urgency=medium * 1.23.4 tagged with 'make release' Commit: Bail out earlier on large tron logs (#881) It's possible for there to be multiple gigabytes of logs in our logging streams and we don't want Tron sitting there processing these for ages on end. This isn't a silver bullet, but this should help in the meantime -- root Mon, 24 Oct 2022 16:02:12 +0000 tron (1.23.3) bionic; urgency=medium * 1.23.3 tagged with 'make release' Commit: TRON-1806 Adds new function in check_tron_jobs to skip failed logging queries for a specific superregion (#880) Adds new function in check_tron_jobs --skip-sensu-failure-logging -- root Fri, 07 Oct 2022 19:53:47 +0000 tron (1.23.2) bionic; urgency=medium * 1.23.2 tagged with 'make release' Commit: fixes incorrect test -- root Thu, 06 Oct 2022 15:24:16 +0000 tron (1.23.1) bionic; urgency=medium * 1.23.1 tagged with 'make release' Commit: Merge pull request #877 from Yelp/jammy-3.7 Python 3.7; build for Jammy -- root Thu, 15 Sep 2022 22:42:37 +0000 tron (1.23.0) bionic; urgency=medium * 1.23.0 tagged with 'make release' Commit: This adds rookout to tron (TRON-1764) (#875) Adds rookout to tron (TRON-1764) -- root Mon, 12 Sep 2022 19:54:31 +0000 tron (1.22.0) bionic; urgency=medium * 1.22.0 tagged with 'make release' Commit: Merge pull request #873 from Yelp/u/emanelsabban/tronUI-1737 Limiting amount of output displayed in Tron UI - TRON-1737 -- root Tue, 23 Aug 2022 19:35:57 +0000 tron (1.21.0) bionic; urgency=medium * 1.21.0 tagged with 'make release' Commit: Merge pull request #874 from Yelp/u/jfong/TRON-1777-update- taskproc TRON-1777: bump taskproc to get kubeconfig reload fix -- root Mon, 15 Aug 2022 18:29:12 +0000 tron (1.20.0) bionic; urgency=medium * 1.20.0 tagged with 'make release' Commit: bumped up taskproc ver pass tron version to tskprc -- root Mon, 25 Jul 2022 20:44:50 +0000 tron (1.19.0) bionic; urgency=medium * 1.19.0 tagged with 'make release' Commit: Pass port/field selector envvars to task_proc (#869) we need this for spark drivers in k8s to work - and without the field selector env vars we're not able to fully adhere to the paasta workload contract -- root Mon, 18 Apr 2022 16:10:56 +0000 tron (1.18.0) bionic; urgency=medium * 1.18.0 tagged with 'make release' Commit: Merge pull request #867 from Yelp/u/kawaiwan/retry-honors- triggers Add arg to for waiting for dependencies -- root Wed, 13 Apr 2022 18:41:37 +0000 tron (1.17.1) bionic; urgency=medium * 1.17.1 tagged with 'make release' Commit: Create KubernetesActionRuns for executor: spark (#866) For now, we should be able to just use a KubernetesActionRun for Spark drivers - worst case, I think we'll just have a couple places where we branch on if we absolutely need to do something Spark-specific and can't just pass that down from paasta-tools This PR also removes spark_driver_service_account_name - I'm not sure what I was thinking initially - we don't really need anything to distinguish between the driver and executor SA since the executor SA is configured by the arguments used to start the driver (and thus we can just start the driver pod as we normally would by using the service account payload that we'd use for a normal tron-launched pod) -- root Mon, 21 Mar 2022 15:03:15 +0000 tron (1.17.0) bionic; urgency=medium * 1.17.0 tagged with 'make release' Commit: Bump pysensu-yelp to pull in issuetype support (#864) https://github.com/Yelp/pysensu-yelp/pull/30 was released in pysensu- yelp==0.4.4 and allows users to set the issuetype for a ticket (so that sensu-created tickets aren't always of type (the default)) -- root Thu, 10 Mar 2022 18:47:54 +0000 tron (1.16.3) bionic; urgency=medium * 1.16.3 tagged with 'make release' Commit: Respect ActionRunAdapter max_lines param (#862) Callers can request a specific number of lines for logs - before this change we were ignoring that (except for the k8s metadata logs) and always returning all the data available. -- root Thu, 10 Feb 2022 22:45:16 +0000 tron (1.16.3) bionic; urgency=medium * 1.16.3 tagged with 'make release' Commit: Respect ActionRunAdapter max_lines param (#862) Callers can request a specific number of lines for logs - before this change we were ignoring that (except for the k8s metadata logs) and always returning all the data available. -- root Thu, 10 Feb 2022 19:46:18 +0000 tron (1.16.2) bionic; urgency=medium * 1.16.2 tagged with 'make release' Commit: Merge pull request #861 from Yelp/u/kawaiwan/always-restart- trond Always restart trond if it goes down -- root Thu, 03 Feb 2022 19:20:28 +0000 tron (1.16.1) xenial; urgency=medium * 1.16.1 tagged with 'make release' Commit: Silence yelp_clog.StreamTailer logs + k8s event guard (#855) This should cleanup any non-fatal exceptions that we see in Tron logs. -- root Tue, 07 Dec 2021 19:39:33 +0000 tron (1.16.0) xenial; urgency=medium * 1.16.0 tagged with 'make release' Commit: Allow users to use runid for triggers (#853) Some users have jobs whose output depends on the output of the previous run - for critical jobs, it's useful to explicitly declare this dependency so that these jobs never run unless Tron knows that the previous output is ready/complete (using successful completion as a proxy for this) For additional context: https://yelp.slack.com/archives/CA4K8PBLG/p1638402768180500 -- root Mon, 06 Dec 2021 19:06:11 +0000 tron (1.15.0) xenial; urgency=medium * 1.15.0 tagged with 'make release' Commit: Add Service Account support for k8s jobs (#852) We'll use this for Pod Identity - there's a webhook that will inject a secret token + some environment variables if a Pod has a Service Account set up. paasta-tools will be in charge of creating the Service Account that we'll use if it doesn't exist (as well as setting up the annotations on that Service Account). -- root Fri, 03 Dec 2021 20:41:48 +0000 tron (1.14.5) xenial; urgency=medium * 1.14.5 tagged with 'make release' Commit: Allow passing Pod annotations to task_processing (#850) We'll need this to toggle certain behaviors (e.g., disable Pods getting routable IPs internally). -- root Thu, 28 Oct 2021 18:11:55 +0000 tron (1.14.4) xenial; urgency=medium * 1.14.4 tagged with 'make release' Commit: Update scribereader to v0.14.1 (#847) v0.14.0 has a bug that points to the wrong hosts (corpdev uses dev infra, not its own) -- root Wed, 06 Oct 2021 19:44:17 +0000 tron (1.14.3) xenial; urgency=medium * 1.14.3 tagged with 'make release' Commit: Upgrade scribereader so that we can read logs in corpdev (#846) We duplicate the ecosystems that we can point scribereader at in the package and that list was missing corpdev This causes a KeyError when trying to get the tailer host:port tuple for tron in corpdev -- root Fri, 01 Oct 2021 19:57:14 +0000 tron (1.14.2) xenial; urgency=medium * 1.14.2 tagged with 'make release' Commit: Support passing Pod labels to task_processing (#841) We'll need this to set labels required by the PaaSTA Workload Contract (which is needed to ensure that internal tooling will work as expected with our Pods) -- root Fri, 10 Sep 2021 17:32:09 +0000 tron (1.14.1) xenial; urgency=medium * 1.14.1 tagged with 'make release' Commit: Show duration in ActionRun history table (#840) This was missed when this view was initially created and is helpful for users to determine how run times for a given action are changing over time at a glance. -- root Thu, 02 Sep 2021 20:42:19 +0000 tron (1.14.0) xenial; urgency=medium * 1.14.0 tagged with 'make release' Commit: Support node selectors/affinity for k8s tasks (#837) This additionally adds some error handling when things fail any invariants for a KubernetesTaskConfig - previously, an InvariantException meant that we would never realize that a KubernetesTaskConfig was invalid and that the ActionRun would never work. -- root Thu, 26 Aug 2021 16:00:51 +0000 tron (1.13.2) xenial; urgency=medium * 1.13.2 tagged with 'make release' Commit: Use start and end times from ActionRunAttempts (#839) It turns out that we reset the start time for an ActionRun on any retries, so let's use the start time from the first attempt and the end time from the last attempt to figure out what timespan to ask for logs from scribereader -- root Tue, 24 Aug 2021 17:55:59 +0000 tron (1.13.1) xenial; urgency=medium * 1.13.1 tagged with 'make release' Commit: Merge pull request #836 from Yelp/jfong/TRON-1658- kubernetesactionrunfromstate TRON-1658: Fix issue restoring KubernetesActionRuns -- root Wed, 18 Aug 2021 17:28:33 +0000 tron (1.13.0) xenial; urgency=medium * 1.13.0 tagged with 'make release' Commit: Merge pull request #830 from Yelp/u/kawaiwan/automate- backfills Make tronctl backfill actually run backfills -- root Tue, 17 Aug 2021 16:38:44 +0000 tron (1.12.0) xenial; urgency=medium * 1.12.0 tagged with 'make release' Commit: Actually kill k8s Pods on KubernetesActionRun::kill() (#828) We had left this method stubbed out until we implemented kill() in task_processing, but now we can actually finish implementation here. -- root Mon, 16 Aug 2021 22:03:29 +0000 tron (1.11.0) xenial; urgency=medium * 1.11.0 tagged with 'make release' Commit: Support adding/dropping capabilities (#827) We'll need this for parity with the Mesos implementation and because it's a good security practice :p -- root Tue, 03 Aug 2021 19:32:46 +0000 tron (1.10.3) xenial; urgency=medium * 1.10.3 tagged with 'make release' Commit: Merge pull request #824 from Yelp/u/kawaiwan/turn-on- taskproc-metrics Install yelp-meteorite when building yelp env -- root Mon, 02 Aug 2021 20:22:57 +0000 tron (1.10.2) xenial; urgency=medium * 1.10.2 tagged with 'make release' Commit: Use un-typo'd failed platform_type for k8s events (#825) see https://github.com/Yelp/task_processing/pull/173 :p -- root Mon, 02 Aug 2021 18:56:46 +0000 tron (1.10.1) xenial; urgency=medium * 1.10.1 tagged with 'make release' Commit: Merge pull request #822 from Yelp/jfong/TRON-1627- secret_env_to_taskproc TRON-1627: Pass secret_env to taskproc from kubernetes -- root Wed, 28 Jul 2021 22:43:21 +0000 tron (1.10.0) xenial; urgency=medium * 1.10.0 tagged with 'make release' Commit: Enable submitting a task to k8s using task_proc (#818) * Enable submitting a task to k8s using task_proc We aren't yet reading any of the events that task_proc is bubbling back up to us, but that'll come next. With this, we should be giving task_proc all the information it needs to actually create a usable Pod. -- root Tue, 27 Jul 2021 20:24:40 +0000 tron (1.9.1) xenial; urgency=medium * 1.9.1 tagged with 'make release' Commit: Release v1.9.0 -- root Mon, 26 Jul 2021 19:30:29 +0000 tron (1.8.1) xenial; urgency=medium * 1.8.1 tagged with 'make release' Commit: Add black for formatting + some additional linters (#805) We don't have any automatically enforced formatting (yapf is optional and not part of our pre-commit), so let's use black since that seems to be what we've settled on. Additionally, I've added some other 'standard' pre-commit hooks (as well as reformatted the pre-commit config file.) -- root Thu, 10 Jun 2021 18:52:48 +0000 tron (1.8.0) xenial; urgency=medium * 1.8.0 tagged with 'make release' Commit: Add initial k8s config + global k8s toggle + job opt-out (#802) We'll be adding two toggles to control Tron's usage of k8s: * a global killswitch (k8s_options['enabled']) for when we want to go back to Mesos for an entire cluster (and, in the future, for when we want to quickly stop all Tronjobs). * a per-job opt-out () for any jobs that encounter issues with k8s or that we want to migrate at a specific time. Since I was adding a k8s config section to the Tron master config, I also went ahead and added a way to configure what the k8s API address should be (i.e., the 'master' address). -- root Wed, 02 Jun 2021 16:01:28 +0000 tron (1.7.0) xenial; urgency=medium * 1.7.0 tagged with 'make release' Commit: Implementation of skip-and-publish (#789) Skipping an action that has downstream triggers is error-prone as its easy to forget that you have triggers to emit and what triggers to emit. You probably need to go check your tronfig definitions to see what triggers exist and converting the trigger name to a trigger command can be a little tricky (with date magic). To solve this, we add a new tronctl command that will skip an action and then publish all triggers for that action. -- root Thu, 15 Apr 2021 17:15:50 +0000 tron (1.6.1) xenial; urgency=medium * 1.6.1 tagged with 'make release' Commit: Merge pull request #780 from Yelp/TRON-1570-remove-old-state Stop saving command config and fields replaced by command config and … -- root Fri, 30 Oct 2020 16:39:41 +0000 tron (1.6.0) xenial; urgency=medium * 1.6.0 tagged with 'make release' Commit: Merge pull request #778 from Yelp/TRON-1161-retry-configs TRON-1161: update configs for retries -- root Wed, 07 Oct 2020 20:23:20 +0000 tron (1.5.1) xenial; urgency=medium * 1.5.1 tagged with 'make release' Commit: Merge pull request #777 from Yelp/only-reconfigure-namespace Only reconfigure jobs in that namespace when a namespace is updated -- root Thu, 01 Oct 2020 18:18:18 +0000 tron (1.5.0) xenial; urgency=medium * 1.5.0 tagged with 'make release' Commit: Merge pull request #775 from Yelp/TRON-1563-retries-separate TRON-1563: save state and configs for retries independently -- root Tue, 15 Sep 2020 00:33:05 +0000 tron (1.4.6) xenial; urgency=medium * 1.4.6 tagged with 'make release' Commit: Merge pull request #774 from Yelp/drmorr/TRON- 1566/redact_aws_keys redact AWS credentials from the logs -- root Wed, 09 Sep 2020 17:44:42 +0000 tron (1.4.5) xenial; urgency=medium * 1.4.5 tagged with 'make release' Commit: Merge pull request #773 from Yelp/TRON-1564-allow-check-oom- events Skip check_oom_events key from monitoring, which is only used by paasta -- root Thu, 03 Sep 2020 17:30:12 +0000 tron (1.4.4) xenial; urgency=medium * 1.4.4 tagged with 'make release' Commit: Merge pull request #769 from Yelp/try-react Prototype React version of tronweb -- root Fri, 28 Aug 2020 18:20:16 +0000 tron (1.4.3) xenial; urgency=medium * 1.4.3 tagged with 'make release' Commit: Merge pull request #770 from Yelp/TRON-1561-no-overlap-alert- if-queueing Do not alert for overlapping runs if queueing is disabled -- root Wed, 26 Aug 2020 21:02:42 +0000 tron (1.4.2) xenial; urgency=medium * 1.4.2 tagged with 'make release' Commit: Merge pull request #765 from Yelp/drmorr/TRON- 1554/fix_pypi_url don't use public pypi for internal builds -- root Tue, 04 Aug 2020 18:23:42 +0000 tron (1.4.1) xenial; urgency=medium * 1.4.1 tagged with 'make release' Commit: v1.4.1 -- root Thu, 30 Jul 2020 23:56:44 +0000 tron (1.4.0) xenial; urgency=medium * 1.4.0 tagged with 'make release' Commit: Merge branch 'TRON-1527-separate-job-run-state' -- root Tue, 28 Jul 2020 22:24:08 +0000 tron (1.3.15) xenial; urgency=medium * 1.3.15 tagged with 'make release' Commit: Merge pull request #757 from Yelp/dedup-save-queue De- duplicate items in the DynamoDB save queue -- root Thu, 23 Jul 2020 20:01:54 +0000 tron (1.3.14) xenial; urgency=medium * 1.3.14 tagged with 'make release' Commit: Merge pull request #756 from Yelp/TRON-1539-dynamo-metrics Tron 1539 dynamo metrics -- root Wed, 01 Jul 2020 21:26:52 +0000 tron (1.3.13) xenial; urgency=medium * 1.3.13 tagged with 'make release' Commit: Merge pull request #755 from Yelp/drmorr/COMPINFRA- 333/bump_requirements Bump requirements to pick up new task_proc version -- root Wed, 06 May 2020 22:55:55 +0000 tron (1.3.12) xenial; urgency=medium * 1.3.12 tagged with 'make release' Commit: Merge pull request #754 from Yelp/u/dpopes/TRON-1531- keyboard-interactive-suppress-tty-if-no-prompt TRON-1531: Suppress the need to prompt the user via tty -- root Tue, 05 May 2020 18:26:53 +0000 tron (1.3.11) xenial; urgency=medium * 1.3.11 tagged with 'make release' Commit: Merge pull request #753 from Yelp/u/dpopes/SEC-12778-support- keyboard-interactive-ssh Support keyboard-interactive as an authentication method for ssh -- root Mon, 27 Apr 2020 18:01:41 +0000 tron (1.3.10) xenial; urgency=medium * 1.3.10 tagged with 'make release' Commit: block new saves if save queue is too big -- root Tue, 14 Apr 2020 16:51:46 +0000 tron (1.3.9) xenial; urgency=medium * 1.3.9 tagged with 'make release' Commit: fix tests, fix issue where stopping condition can't be reached -- root Tue, 14 Apr 2020 14:46:43 +0000 tron (1.3.8) xenial; urgency=medium * 1.3.8 tagged with 'make release' Commit: consume save queue in predefined chunks, count errors correctly -- root Tue, 14 Apr 2020 14:07:12 +0000 tron (1.3.7) xenial; urgency=medium * 1.3.7 tagged with 'make release' Commit: Merge pull request #749 from Yelp/u/maksym/fix-redirect Fix redirect, comment out stuff in tronrepl -- root Wed, 08 Apr 2020 23:17:14 +0000 tron (1.3.6) xenial; urgency=medium * 1.3.6 tagged with 'make release' Commit: Merge pull request #745 from Yelp/dependabot/pip/psutil- 5.6.6 Bump psutil from 5.6.3 to 5.6.6 -- root Thu, 02 Apr 2020 23:56:54 +0000 tron (1.3.5) xenial; urgency=medium * 1.3.5 tagged with 'make release' Commit: Merge pull request #744 from Yelp/mbehrens-TRON-1144-add- uptime-version Add tron version and uptime to frontend navbar -- root Mon, 16 Mar 2020 17:38:19 +0000 tron (1.3.4) xenial; urgency=medium * 1.3.4 tagged with 'make release' Commit: Merge pull request #741 from Yelp/fix-tz-forward Fix bug from fall forward last year -- root Fri, 06 Mar 2020 18:59:59 +0000 tron (1.3.3) xenial; urgency=medium * 1.3.3 tagged with 'make release' Commit: Merge pull request #737 from Yelp/u/siruitan/TRON-385- skip_validate_in_write_config Skip job graph validation in write_config -- root Thu, 06 Feb 2020 18:59:31 +0000 tron (1.3.2) xenial; urgency=medium * 1.3.2 tagged with 'make release' Commit: Merge branch 'drmorr/TRON- 1369/fixing_the_clusterman_invocation' -- root Wed, 05 Feb 2020 23:32:42 +0000 tron (1.3.1) xenial; urgency=medium * 1.3.1 tagged with 'make release' Commit: Merge pull request #735 from Yelp/drmorr/TRON- 1454/shorten_tron_directory_names tron output dir names shortened -- root Tue, 04 Feb 2020 18:35:32 +0000 tron (1.3.0) xenial; urgency=medium * 1.3.0 tagged with 'make release' Commit: Merge branch 'drmorr/TRON-1369/clusterman_env_var' -- root Thu, 16 Jan 2020 18:59:08 +0000 tron (1.2.5) xenial; urgency=medium * 1.2.5 tagged with 'make release' Commit: Merge branch 'u/kawaiwan/bump-task-proc-to-0.1.8' -- root Tue, 12 Nov 2019 01:01:46 +0000 tron (1.2.4) xenial; urgency=medium * 1.2.4 tagged with 'make release' Commit: Merge pull request #723 from Yelp/drmorr/TRON- 1329/fix_time_interval_construction fix time interval construction in check_tron_jobs -- root Wed, 06 Nov 2019 21:56:14 +0000 tron (1.2.3) xenial; urgency=medium * 1.2.3 tagged with 'make release' Commit: Merge pull request #717 from Yelp/monitoring-backfill- buckets Try to handle backfills better in monitoring precious runs -- root Tue, 15 Oct 2019 23:27:13 +0000 tron (1.2.2) xenial; urgency=medium * 1.2.2 tagged with 'make release' Commit: Merge branch 'drmorr/TRON- 1303/action_runner_sets_env_variables' -- root Mon, 14 Oct 2019 19:47:54 +0000 tron (1.2.1) xenial; urgency=medium * 1.2.1 tagged with 'make release' Commit: Merge pull request #716 from Yelp/retry-check-state Do not auto retry action if state is already succeeded -- root Wed, 09 Oct 2019 16:57:32 +0000 tron (1.2.0) xenial; urgency=medium * 1.2.0 tagged with 'make release' Commit: Merge pull request #715 from Yelp/auto-recovery Automatically try to recover on unknown ssh actions -- root Wed, 09 Oct 2019 00:17:59 +0000 tron (1.1.0) xenial; urgency=medium * 1.1.0 tagged with 'make release' Commit: Merge pull request #714 from Yelp/recover-command Add manual recovery command to api -- root Thu, 03 Oct 2019 16:54:12 +0000 tron (1.0.4) xenial; urgency=medium * 1.0.4 tagged with 'make release' Commit: Merge pull request #708 from Yelp/recovery-check-finished Put back check for action runners that have completed in recovery -- root Tue, 24 Sep 2019 00:50:16 +0000 tron (1.0.3) xenial; urgency=medium * 1.0.3 tagged with 'make release' Commit: Merge pull request #706 from Yelp/fix_check_tron_jobs_job_run_ids fix job run ids reported by check_tron_jobs -- root Thu, 19 Sep 2019 21:56:23 +0000 tron (1.0.2) xenial; urgency=medium * 1.0.2 tagged with 'make release' Commit: Don't do a retry_delay when manually retrying a failed command. (#704) -- root Wed, 18 Sep 2019 20:28:54 +0000 tron (1.0.1) xenial; urgency=medium * 1.0.1 tagged with 'make release' Commit: Merge pull request #692 from Yelp/taskproc_bump Bump taskproc to handle weird staging offers -- root Wed, 07 Aug 2019 13:37:07 +0000 tron (1.0.0) xenial; urgency=medium * 1.0.0 tagged with 'make release' Commit: Released 0.9.14.6 via make release -- root Fri, 19 Jul 2019 23:23:34 +0000 tron (0.9.14.6) xenial; urgency=medium * 0.9.14.6 tagged with 'make release' Commit: Merge pull request #688 from Yelp/sort_order_fixes Make string sorting only in ActionRun displaying. -- root Tue, 16 Jul 2019 21:46:48 +0000 tron (0.9.14.5) xenial; urgency=medium * 0.9.14.5 tagged with 'make release' Commit: Released 0.9.14.4 via make release -- root Tue, 16 Jul 2019 18:23:28 +0000 tron (0.9.14.4) xenial; urgency=medium * 0.9.14.4 tagged with 'make release' Commit: Merge pull request #687 from Yelp/fix_tron_sorting Fix sorting for DisplayActionRuns when fields are None -- root Tue, 16 Jul 2019 18:21:47 +0000 tron (0.9.14.4) xenial; urgency=medium * 0.9.14.4 tagged with 'make release' Commit: Merge pull request #682 from Yelp/stderr_fix Fix stderr output on check_tron_jobs -- root Fri, 12 Jul 2019 16:56:55 +0000 tron (0.9.14.3) xenial; urgency=medium * 0.9.14.3 tagged with 'make release' Commit: Merge pull request #681 from Yelp/refix-scheduling Use last run time to get next run time before scheduling -- root Thu, 11 Jul 2019 18:11:28 +0000 tron (0.9.14.2) xenial; urgency=medium * 0.9.14.2 tagged with 'make release' Commit: Merge pull request #678 from Yelp/ssh-timeouts Improve logging for deferred errors and apply timeout config to trans… -- root Fri, 05 Jul 2019 20:32:44 +0000 tron (0.9.14.1) xenial; urgency=medium * 0.9.14.1 tagged with 'make release' Commit: Merge branch 'master' of github.com:Yelp/Tron -- root Mon, 03 Jun 2019 19:01:33 +0000 tron (0.9.14.0) xenial; urgency=medium * 0.9.14.0 tagged with 'make release' Commit: Merge pull request #668 from Yelp/another-waiting-fix Runs with triggers should start in scheduled state -- root Thu, 16 May 2019 17:30:06 +0000 tron (0.9.13.4) xenial; urgency=medium * 0.9.13.4 tagged with 'make release' Commit: Merge pull request #667 from Yelp/revert-665-revert-654- u/mingqiz/disable_shelvestore removed mirror store -- root Tue, 14 May 2019 23:09:44 +0000 tron (0.9.13.3) xenial; urgency=medium * 0.9.13.3 tagged with 'make release' Commit: Merge pull request #666 from Yelp/u/kawaiwan/improve- recovery Make recovery batch script also check if action runner suddenly goes away -- root Tue, 14 May 2019 00:08:31 +0000 tron (0.9.13.2) xenial; urgency=medium * 0.9.13.2 tagged with 'make release' Commit: fix guess_realert when next_run is the same as previous_run (#658) fix guess_realert when next_run is the same as previous_run -- root Thu, 02 May 2019 17:48:01 +0000 tron (0.9.13.1) xenial; urgency=medium * 0.9.13.1 tagged with 'make release' Commit: Merge pull request #656 from Yelp/u/mingqiz/fix_dynamodb_bug Changed dynamodb partition index to int so it is sorted correctly -- root Tue, 30 Apr 2019 20:05:19 +0000 tron (0.9.13.0) xenial; urgency=medium * 0.9.13.0 tagged with 'make release' Commit: Merge pull request #649 from Yelp/drmorr/TRON- 1095/add_duration_to_tronweb duration field added to tronweb -- root Mon, 22 Apr 2019 23:16:52 +0000 tron (0.9.12.6) xenial; urgency=medium * 0.9.12.6 tagged with 'make release' Commit: Merge pull request #648 from Yelp/u/mingqiz/TRON-1088-bin Removed interval scheduler -- root Fri, 19 Apr 2019 18:11:18 +0000 tron (0.9.12.5) xenial; urgency=medium * 0.9.12.5 tagged with 'make release' Commit: Merge branch 'drmorr/TRON-605/fix_monitoring_for_stuck_jobs' -- root Mon, 15 Apr 2019 21:02:28 +0000 tron (0.9.12.4) xenial; urgency=medium * 0.9.12.4 tagged with 'make release' Commit: Merge pull request #640 from Yelp/u/mingqiz/TRON-638- improve_read_speed Improve dynamodb read/write speed -- root Wed, 10 Apr 2019 22:59:47 +0000 tron (0.9.12.3) xenial; urgency=medium * 0.9.12.3 tagged with 'make release' Commit: Merge pull request #639 from Yelp/u/mingqiz/TRON-1041- open_file_limit changed max number of open files to 10000 -- root Wed, 03 Apr 2019 00:39:28 +0000 tron (0.9.12.2) xenial; urgency=medium * 0.9.12.2 tagged with 'make release' Commit: tron/__init__.py -- root Tue, 02 Apr 2019 18:10:28 +0000 tron (0.9.12.1) xenial; urgency=medium * 0.9.12.1 tagged with 'make release' Commit: Merge pull request #631 from Yelp/u/mingqiz/TRON-638- fix_validation fixed data validation of dynamodb migration -- root Fri, 22 Mar 2019 18:06:11 +0000 tron (0.9.12.0) xenial; urgency=medium * 0.9.12.0 tagged with 'make release' Commit: Merge branch 'drmorr/TRON-390/xjob_dep_viz' -- root Wed, 13 Mar 2019 18:39:40 +0000 tron (0.9.11.1) xenial; urgency=medium * 0.9.11.1 tagged with 'make release' Commit: Merge pull request #628 from Yelp/u/mingqiz/TRON-661 Fixed trusty build -- root Tue, 12 Mar 2019 00:32:40 +0000 tron (0.9.11.0) xenial; urgency=medium * 0.9.11.0 tagged with 'make release' Commit: Merge pull request #617 from Yelp/u/mingqiz/TRON-638- migration Migrating from Berkley DB to DynamoDB (TRON-638) -- root Fri, 08 Mar 2019 17:27:53 +0000 tron (0.9.10.0) xenial; urgency=medium * 0.9.10.0 tagged with 'make release' Commit: Merge pull request #622 from Yelp/waiting-state Waiting state if an action is waiting for normal or cross-job dependencies -- root Mon, 04 Mar 2019 19:48:20 +0000 tron (0.9.9.15) xenial; urgency=medium * 0.9.9.15 tagged with 'make release' Commit: Merge pull request #624 from Yelp/add-waiting-state Add waiting state first for rollback safety -- root Mon, 04 Mar 2019 17:53:07 +0000 tron (0.9.9.14) trusty; urgency=medium * 0.9.9.14 tagged with 'make release' Commit: Merge pull request #615 from Yelp/fi'-check-job-duration Ignore duration=None for jobs waiting on e'ternal dependency -- root Fri, 08 Feb 2019 12:31:54 +0000 tron (0.9.9.13) trusty; urgency=medium * 0.9.9.13 tagged with 'make release' Commit: fi' disk=none error when restoring from state -- root Thu, 07 Feb 2019 16:18:18 +0000 tron (0.9.9.12) trusty; urgency=medium * 0.9.9.12 tagged with 'make release' Commit: Merge pull request #613 from Yelp/fix-jobrun-state-proxy Fix inconsistency between JobRun state attribute and is_ checks -- root Thu, 07 Feb 2019 15:14:29 +0000 tron (0.9.9.11) trusty; urgency=medium * 0.9.9.11 tagged with 'make release' Commit: Merge pull request #610 from Yelp/disk_support Added disk support to tron on mesos -- root Mon, 04 Feb 2019 18:50:33 +0000 tron (0.9.9.10) trusty; urgency=medium * 0.9.9.10 tagged with 'make release' Commit: Merge pull request #611 from Yelp/fix-job-appearing-pending Fix job appearing pending when waiting on trigger requirement -- root Thu, 31 Jan 2019 11:44:45 +0000 tron (0.9.9.9) trusty; urgency=medium * 0.9.9.9 tagged with 'make release' Commit: tron.config.schema: use enum types values e'plicitly to fi' regression after migrating to native enums -- root Mon, 07 Jan 2019 13:46:43 +0000 tron (0.9.9.8) trusty; urgency=medium * 0.9.9.8 tagged with 'make release' Commit: Merge pull request #603 from Yelp/recovery-none-action-runs Skip job runs with no action runs during recovery -- root Thu, 20 Dec 2018 19:19:02 +0000 tron (0.9.9.7) trusty; urgency=medium * 0.9.9.7 tagged with 'make release' * Fix type bug in creating tasks during recovery * Only recover Mesos actions with no end time -- root Tue, 18 Dec 2018 19:27:22 +0000 tron (0.9.9.6) trusty; urgency=medium * 0.9.9.6 tagged with 'make release' * Fixes for inactive framework -- root Thu, 06 Dec 2018 00:59:04 +0000 tron (0.9.9.5) trusty; urgency=medium * 0.9.9.5 tagged with 'make release' * Allow passing extra options to systemd-based distros. * Update end times for actions * Fix bug related to Mesos offer timeouts * Do not reschedule disabled job if it is reconfigured -- root Tue, 04 Dec 2018 21:38:20 +0000 tron (0.9.9.4) trusty; urgency=medium * 0.9.9.4 tagged with 'make release' - Fix signal handling so that Tron gracefull shuts down - Add more information to log lines -- root Mon, 19 Nov 2018 23:59:43 +0000 tron (0.9.9.3) trusty; urgency=medium * 0.9.9.3 tagged with 'make release' Commit: Merge pull request #587 from Yelp/TRON-536 Removed spaces in action run html -- root Thu, 15 Nov 2018 00:20:45 +0000 tron (0.9.9.2) trusty; urgency=medium * 0.9.9.2 tagged with 'make release' Commit: Merge pull request #583 from Yelp/u/kawaiwan/add-cluster-to- tron-metrics Add cluster option to metrics script -- root Mon, 12 Nov 2018 18:25:34 +0000 tron (0.9.9.1) trusty; urgency=medium * 0.9.9.1 tagged with 'make release' Commit: modify action_id during job migration (#580) * modify action_id during job migration * pin pre-commit 1.11.2 -- root Tue, 30 Oct 2018 21:42:21 +0000 tron (0.9.9.0) trusty; urgency=medium * 0.9.9.0 tagged with 'make release' * Tronctl publish/discard events * Update DST fall back behavior to be consistent * Update requirements * Add script for fetching metrics -- root Tue, 30 Oct 2018 17:54:04 +0000 tron (0.9.8.4) trusty; urgency=medium * 0.9.8.4 tagged with 'make release' Commit: Revert 'Merge pull request #570 from Yelp/pin-requirements' This reverts commit 33fb19df2d2ec508d254f209c1515bbffabc1523, reversing changes made to b3c7ab8ae8ffa2f7a08e275175989ea5d8d54a8a. -- root Fri, 26 Oct 2018 21:36:49 +0000 tron (0.9.8.3) trusty; urgency=medium * 0.9.8.3 tagged with 'make release' Commit: Merge pull request #576 from Yelp/yapf-it yapf -rip -- root Fri, 26 Oct 2018 17:16:09 +0000 tron (0.9.8.2) trusty; urgency=medium * 0.9.8.2 tagged with 'make release' Commit: Merge pull request #576 from Yelp/yapf-it yapf -rip -- root Fri, 26 Oct 2018 17:14:02 +0000 tron (0.9.8.2) trusty; urgency=medium * 0.9.8.2 tagged with 'make release' Commit: Merge pull request #574 from Yelp/u/chl/rename_all_job_name_after_migration rename all job_name after migration -- root Thu, 25 Oct 2018 22:38:54 +0000 tron (0.9.8.1) trusty; urgency=medium * 0.9.8.1 tagged with 'make release' Commit: Merge pull request #569 from Yelp/signal-none Don't log other signals -- root Tue, 23 Oct 2018 23:30:47 +0000 tron (0.9.8.0) trusty; urgency=medium * 0.9.8.0 tagged with 'make release' Commit: Merge pull request #566 from Yelp/monitoring-allow-overlap If runs are allowed to overlap, don't consider that case stuck -- root Tue, 23 Oct 2018 20:51:02 +0000 tron (0.9.7.0) trusty; urgency=medium * 0.9.7.0 tagged with 'make release' Commit: fix typo in systemd unit file -- root Wed, 17 Oct 2018 17:37:58 +0000 tron (0.9.6.5) trusty; urgency=medium * 0.9.6.5 tagged with 'make release' Commit: Merge pull request #561 from Yelp/fix-dependent Fix actiongraph adapter failing to render dependent actions -- root Tue, 16 Oct 2018 17:37:45 +0000 tron (0.9.6.4) trusty; urgency=medium * 0.9.6.4 tagged with 'make release' Commit: Released 0.9.6.3 via make release -- root Mon, 15 Oct 2018 18:27:07 +0000 tron (0.9.6.3) trusty; urgency=medium * 0.9.6.3 tagged with 'make release' Commit: Merge pull request #555 from Yelp/u/chl/TRON- 442_create_migrate_job_tool add tronctl move command -- root Mon, 15 Oct 2018 17:52:20 +0000 tron (0.9.6.2) trusty; urgency=medium * 0.9.6.2 tagged with 'make release' Commit: Merge pull request #548 from Yelp/skip-docs-diagram Skip state diagram in docs -- root Tue, 02 Oct 2018 12:23:05 +0000 tron (0.9.6.1) trusty; urgency=medium * 0.9.6.1 tagged with 'make release' Commit: fix format string bugs (#538) * fix format string bugs -- root Fri, 21 Sep 2018 23:46:13 +0000 tron (0.9.6.0) trusty; urgency=medium * 0.9.6.0 tagged with 'make release' * Remove percent string support * Fix bug with fail and retries * Remove headers from tronfig * Emit triggers for cross-job dependencies -- root Thu, 20 Sep 2018 01:20:15 +0000 tron (0.9.5.1) trusty; urgency=medium * 0.9.5.1 tagged with 'make release' * Update taskproc to 0.1.2 for Mesos fixes * Save jobs when re-configured * Fix tronweb CSS * Make scheme optional for Mesos master address * More feedback on killing Mesos actions -- root Mon, 10 Sep 2018 21:14:06 +0000 tron (0.9.5.0) trusty; urgency=medium * 0.9.5.0 tagged with 'make release' * Increase upstart timeout * Remove enableall/disableall from jobs controller * Deprecate --nodaemon -- root Wed, 05 Sep 2018 22:04:23 +0000 tron (0.9.4.0) trusty; urgency=medium * 0.9.4.0 tagged with 'make release' * add string format support (#490) * recover Mesos action runs on restart * run reactor on separate thread * job trigger configs and eventbus -- root Tue, 04 Sep 2018 22:23:25 +0000 tron (0.9.3.0) trusty; urgency=medium * 0.9.3.0 tagged with 'make release' Commit: make master address optional (#513) -- root Fri, 24 Aug 2018 22:42:36 +0000 tron (0.9.2.1) trusty; urgency=medium * 0.9.2.1 tagged with 'make release' Commit: Merge pull request #508 from Yelp/nix_testify Nix testify -- root Wed, 22 Aug 2018 10:53:38 +0000 tron (0.9.2.0) trusty; urgency=medium * 0.9.2.0 tagged with 'make release' Commit: catch command rendering type error (#488) * catch command rendering type error -- root Thu, 09 Aug 2018 20:46:45 +0000 tron (0.9.1.9) trusty; urgency=medium * 0.9.1.9 tagged with 'make release' Commit: Merge pull request #483 from Yelp/retries-delay-kill Retries delay: kill delayed action correctly -- root Tue, 24 Jul 2018 11:00:12 +0000 tron (0.9.1.8) trusty; urgency=medium * 0.9.1.8 tagged with 'make release' * fix _get_seconds_from_duration bug in monitoring -- root Tue, 10 Jul 2018 20:49:53 +0000 tron (0.9.1.7) trusty; urgency=medium * 0.9.1.7 tagged with 'make release' * Make unknown alerts critical instead of warning * Add manhole for debugging * Fix validation of full tronfig directory -- root Mon, 09 Jul 2018 17:35:38 +0000 tron (0.9.1.6) trusty; urgency=medium * 0.9.1.6 tagged with 'make release' * Bug fixes * Remove graceful shutdown * Mesos: Add default volume configs, implement kill/stop commmands -- root Tue, 03 Jul 2018 17:08:51 +0000 tron (0.9.1.5) trusty; urgency=medium * 0.9.1.5 tagged with 'make release' Commit: Merge pull request #470 from Yelp/fix-output-dir Check output dir first -- root Mon, 25 Jun 2018 19:27:00 +0000 tron (0.9.1.4) trusty; urgency=medium * 0.9.1.4 tagged with 'make release' Commit: Merge pull request #469 from Yelp/u/robj/improve-action- runner action_runner logs to the output_dir; add timestamps to logs -- root Mon, 25 Jun 2018 17:00:57 +0000 tron (0.9.1.3) trusty; urgency=medium * 0.9.1.3 tagged with 'make release' Commit: Merge pull request #468 from Yelp/u/robj/keep-fs-reasders- alive handle failures streaming to stdout/stderr -- root Fri, 22 Jun 2018 17:21:54 +0000 tron (0.9.1.2) trusty; urgency=medium * 0.9.1.2 tagged with 'make release' Commit: Merge pull request #462 from Yelp/missing-deps add missing requests and psutil deps -- root Wed, 20 Jun 2018 17:56:19 +0000 tron (0.9.1.1) trusty; urgency=medium * 0.9.1.1 tagged with 'make release' Commit: bump version -- root Fri, 15 Jun 2018 09:34:03 +0000 tron (0.9.1.0) trusty; urgency=medium * 0.9.1.0 tagged with 'make release' Commit: Merge pull request #452 from Yelp/mesos-logging Get output from Mesos tasks -- root Wed, 13 Jun 2018 13:48:43 +0000 tron (0.9.0.0) trusty; urgency=medium * 0.9.0.0 tagged with 'make release' Commit: Merge pull request #451 from Yelp/fix-machine-state-during- recovery set the machine state to running before recovery -- root Tue, 05 Jun 2018 16:48:52 +0000 tron (0.8.0.6) trusty; urgency=medium * 0.8.0.6 tagged with 'make release' * Support for expected runtime alerts * Pre calculate state machine transitions * Bug fixes -- root Wed, 16 May 2018 18:31:23 +0000 tron (0.8.0.5) trusty; urgency=medium * 0.8.0.5 tagged with 'make release' Commit: Merge pull request #435 from Yelp/encode-stdout maybe_encode all data in the file serializer -- root Tue, 24 Apr 2018 02:46:08 +0000 tron (0.8.0.4) trusty; urgency=medium * 0.8.0.4 tagged with 'make release' Commit: friendlier output from tronctl retry -- root Fri, 20 Apr 2018 14:15:05 +0000 tron (0.8.0.3) trusty; urgency=medium * 0.8.0.3 tagged with 'make release' Feature: Job actions can now be re-tried using cli command `tronctl retry `. This will automatically trigger dependent actions upon success of re-tried action. -- root Thu, 19 Apr 2018 13:14:04 +0000 tron (0.8.0.2) trusty; urgency=medium * 0.8.0.2 tagged with 'make release' Fix: regression in tronweb jobs list -- root Wed, 18 Apr 2018 11:09:19 +0000 tron (0.8.0.1) trusty; urgency=medium * 0.8.0.1 tagged with 'make release' Commit: Merge pull request #370 from Yelp/python3-deb Python3 deb -- root Mon, 16 Apr 2018 14:38:29 +0000 tron (0.8.0.0) trusty; urgency=medium * 0.8.0.0 tagged with 'make release' Commit: remove duplicate ignore for debian/debhelper-build-stamp -- root Wed, 14 Mar 2018 11:14:08 +0000 tron (0.7.8.3) trusty; urgency=medium * 0.7.8.3 tagged with 'make release' Commit: Merge pull request #426 from Yelp/maybe-decode-all-the- things Maybe decode all the things -- root Thu, 12 Apr 2018 13:29:50 +0000 tron (0.7.8.2) trusty; urgency=medium * 0.7.8.2 tagged with 'make release' Commit: Merge pull request #424 from Yelp/cleanup-retries-validation Fix validation of cleanup action -- root Tue, 10 Apr 2018 11:52:29 +0000 tron (0.7.8.1) trusty; urgency=medium * 0.7.8.1 tagged with 'make release' * Retries attribute for action runs * Preparing for Python 3 upgrade -- root Fri, 06 Apr 2018 14:45:12 +0000 tron (0.7.8.0) trusty; urgency=medium * 0.7.8.0 tagged with 'make release' * Script to clean up namespaces * Improve check_tron_jobs logging * Fix bug in date context math with timezones * Improve tab completion * Remove more service code * Use config values to create PaaSTA action run that prints -- root Tue, 03 Apr 2018 18:51:45 +0000 tron (0.7.7.1) trusty; urgency=medium * 0.7.7.1 tagged with 'make release' Commit: Preparing 0.7.7.1 release - fix shelve regression - use bsddb3 directly - remove service functionality from tronweb - add CORS header - dockerize itests - test debian package in travis - tab completion improvements -- root Fri, 23 Mar 2018 21:11:20 +0000 tron (0.7.7.0) trusty; urgency=medium * 0.7.7.0 tagged with 'make release' * Cache job names in tab completion * Remove core service class (services are deprecated now) * Add backward compatible shelve * Add initial config fields for actions on PaaSTA * Bug fixes in tronfig and DST time resolution -- root Thu, 22 Mar 2018 18:33:40 +0000 tron (0.7.6.1) trusty; urgency=medium * 0.7.6.1 tagged with 'make release' Commit: bump version, fix package building issues -- root Mon, 12 Mar 2018 17:08:34 +0000 tron (0.7.6.0) trusty; urgency=medium * 0.7.6.0 tagged with 'make release' Commit: version bump 0.7.6.0 -- root Mon, 12 Mar 2018 15:22:58 +0000 tron (0.7.5.3) trusty; urgency=medium * 0.7.5.3 tagged with 'make release' Commit: Merge pull request #372 from Yelp/u/jgl/TRON- 212_upgrade_to_argparse Upgrade to argparse -- root Thu, 08 Mar 2018 00:39:13 +0000 tron (0.7.5.2) trusty; urgency=medium * 0.7.5.2 tagged with 'make release' Commit: Merge pull request #373 from Yelp/fix_tz_naive_localization Only localize datetimes when they lack tzinfo -- root Fri, 02 Mar 2018 20:49:06 +0000 tron (0.7.5.1) trusty; urgency=medium * 0.7.5.1 tagged with 'make release' Commit: Merge pull request #367 from Yelp/u/jgl/better_pidfile_error_message Make tron pidfile error message more clear -- root Wed, 28 Feb 2018 23:12:22 +0000 tron (0.7.5.0) trusty; urgency=medium * 0.7.5.0 tagged with 'make release' Commit: Merge pull request #360 from Yelp/per_job_tz Allow jobs to override the default timezone -- root Wed, 28 Feb 2018 02:07:09 +0000 tron (0.7.4.2) trusty; urgency=medium * 0.7.4.2 tagged with 'make release' Commit: added xenial building support -- root Tue, 27 Feb 2018 01:23:33 +0000 tron (0.7.4.1) trusty; urgency=medium * 0.7.4.1 tagged with 'make release' Commit: Released 0.7.4.1 via make release -- root Fri, 23 Feb 2018 23:35:28 +0000 tron (0.7.4.0) trusty; urgency=medium * 0.7.4.0 tagged with 'make release' * Remove support for mongodb in state serialization * Remove deprecated restart_interval option for services * Fix unicode bug in root URL -- root Tue, 13 Feb 2018 20:57:16 +0000 tron (0.7.3.2) trusty; urgency=medium * 0.7.3.2 tagged with 'make release' Commit: Merge pull request #347 from Yelp/twisted-twisted Twisted fix, example cluster and itest improvements -- root Fri, 09 Feb 2018 14:39:45 +0000 tron (0.7.3.1) trusty; urgency=medium * 0.7.3.1 tagged with 'make release' Commit: Merge branch dont-start-on-boot -- root Fri, 09 Feb 2018 09:27:12 +0000 tron (0.7.3.0) trusty; urgency=medium * $0.7.3.0 tagged with \make release'rCommit: Merge pull request -- root Thu, 08 Feb 2018 22:42:17 +0000 tron (0.7.2.0) trusty; urgency=medium * 0.7.2.0 tagged with make release' * Use upstart instead of sysv-init * Added prototype check_tron_jobs and monitoring configs * Add --delete option for tronfig namespaces -- root Thu, 01 Feb 2018 11:07:08 +0000 tron (0.7.1.0) trusty; urgency=medium * 0.7.1.0 tagged with 'make release' Commit: dont assume the USER env var use the more reliable getpass.getuser() instead of expecting a USER env var to be present. -- root Tue, 10 Oct 2017 12:47:50 +0000 tron (0.7.0.0) trusty; urgency=medium * 0.7.0.0 tagged with 'make release' Commit: fix init.d script -- root Fri, 25 Aug 2017 14:22:47 +0000 tron (0.6.2.1) lucid; urgency=low * Only keep last buffer from ssh connection -- Federico Giraud Mon, 15 Aug 2016 10:15:56 -0700 tron (0.6.1.12) lucid; urgency=low * Fix memory leaks from event recorder and twisted -- Yejun Yang Wed, 06 Jan 2016 18:07:37 -0800 tron (0.6.1.11) lucid; urgency=low * Add job and service support fields: owner, summary, notes -- Yejun Yang Fri, 02 Oct 2015 11:38:26 -0700 tron (0.6.1.10) lucid; urgency=low * Optimize tronweb dashboard performance -- Yejun Yang Fri, 19 Dec 2014 13:45:23 -0800 tron (0.6.1.9) lucid; urgency=low * Log some known exceptions * Check overlapped run id with instance -- Yejun Yang Thu, 11 Dec 2014 11:54:32 -0800 tron (0.6.1.8) lucid; urgency=low * Fix service instance restore state, run monitor instead of queue. -- Yejun Yang Tue, 04 Nov 2014 14:14:29 -0800 tron (0.6.1.7) lucid; urgency=low * Display error message when instance start fail -- Yejun Yang Tue, 04 Nov 2014 10:18:12 -0800 tron (0.6.1.6) lucid; urgency=low * Ignore service instance start error * Ignore duplicated run id -- Yejun Yang Mon, 03 Nov 2014 17:21:24 -0800 tron (0.6.1.5) lucid; urgency=low * Increase channel start timeout * Fix service monitor restart too soon -- Yejun Yang Thu, 03 Jul 2014 10:28:45 -0700 tron (0.6.1.4) lucid; urgency=low * Remove incorrectly fixed dead code * Service monitor task always notify failed instead of down -- Yejun Yang Tue, 01 Apr 2014 10:43:48 -0700 tron (0.6.1.3) lucid; urgency=low * Fix bug in node service stop -- Yejun Yang Thu, 27 Feb 2014 17:43:57 -0800 tron (0.6.1.2) lucid; urgency=low * Fix bug prevent reconnection * Add new config monitor_retries -- Yejun Yang Wed, 05 Feb 2014 10:29:59 -0800 tron (0.6.1) unstable; urgency=low * tronweb was replaced with a clientside version * more ssh options are now configurable * adding an experimental feature to support a max_runtime on jobs * adding tronctl kill to SIGKILL a service * add a `--no-header` option to tronfig -- Daniel Nephin Thu, 02 May 2013 17:34:46 -0700 tron (0.6.0.2) unstable; urgency=low * Allow serviceinstances to transition from unknown to down * Better handling for serviceinstance monitor task failing -- Daniel Nephin Thu, 04 Apr 2013 12:47:14 -0700 tron (0.6.0.1) unstable; urgency=low * minor visual improvements to tronview -- Daniel Nephin Tue, 26 Mar 2013 12:22:38 -0700 tron (0.6.0) unstable; urgency=low * action.requires must be a list (string has been deprecated since 0.3.3) * tronctl zap has been removed (it shouldn't be necessary anymore) * service monitoring code has been re-written (services should not longer get stuck in a stopping state) * hosts can not be validated by specifying a known_hosts file * additional validation for ssh options and context variables has been moved into configuration validation * tronview now displays additional details about jobs and services -- Daniel Nephin Mon, 25 Mar 2013 11:14:13 -0700 tron (0.5.2.3) unstable; urgency=low * Fix a bug that was preventing nodes from connecting with provided username * Patched an issue with the SSH connection that could cause exceptions on channel close -- Daniel Nephin Fri, 15 Feb 2013 11:19:25 -0800 tron (0.5.2) unstable; urgency=low * Tron now supports the ability to use different users per node connection. * Fragmented configuration is now possible by using namespaced config files. * Additional cleanup and stability patches have been applied. * State persistence configuration can now be changed without restarting trond * State saving now includes a namespace, you will need to run `tools/migration/migrate_state.py` to migrate old state. -- Thomas Robinson Wed, 9 Jan 2013 16:25:53 -0700 tron (0.5.1) unstable; urgency=low * Jobs which are disabled will no longer be re-enabled when part of their configuration changes. * Individual actions for a Job can no longer be started independently before a job is started. This was never intentionally supported. * Adding a new configuration option `allow_overlap` for Jobs, which allows job runs to overlap each other. * Jobs can now be configured using crontab syntax. -- Daniel Nephin Wed, 25 Jul 2012 16:25:53 -0700 tron (0.5.0.2) unstable; urgency=low * Fix a bug with daemonizing and some versions of twisted reactor. -- Daniel Nephin Tue, 17 Jul 2012 19:21:39 -0700 tron (0.5.0) unstable; urgency=low * Names for nodes, jobs, actions and service can now contain underscore characters but are restricted to 255 characters. * trond now supports a graceful shutdown. Send trond SIGINT to have it wait for all currently running jobs to complete before shutting down. SIGTERM also performs some cleanup before terminating. * State serialization has changed. See :ref:`config_state` for configuration options. `tools/migration/migrate_state.py` is included to migrate your existing Tron state to a new store. YAML store is now deprecated. * Old style config, which was deprecated in 0.3 will no longer work. -- Daniel Nephin Tue, 05 Jun 2012 18:47:34 -0700 tron (0.4.1) unstable; urgency=low * tronview will once again attempt to find the tty width even when stdout is not a tty. * Fixed last_success for job context. * Job runs which are manually cancelled will now continue to schedule new runs. -- Daniel Nephin Wed, 30 May 2012 16:35:44 -0700 tron (0.4.0) unstable; urgency=low * Jobs now continue to run all possible actions after one of its actions fail * Enabling a disabled job now schedules the next run using current time instead of the last successful run (which could cause many runs to be scheduled in the past if the job had been disabled for a while) * Resolved many inconsistencies and bugs around Job scheduling. -- Daniel Nephin Fri, 11 May 2012 18:00:00 -0800 tron (0.3.3-1) unstable; urgency=low * Remove logrotate script from debian packaging * Add logging.conf to debian packaging -- James Brown Thu, 19 Apr 2012 14:33:17 -0700 tron (0.3.3) unstable; urgency=low * Adding a configuration migration script for porting 0.2.x configs to the new 0.3.x * Remove working_dir from the configuration and replace with output_stream_dir * Remove logging confiruation from the general config. Logging is now configured using python standaring logging -- Daniel Nephin Wed, 18 Apr 2012 18:00:00 -0800 tron (0.3.2) unstable; urgency=low * Fixes a bug when there are multiple node pools * Adds more unit tests -- Daniel Nephin Wed, 11 Apr 2012 11:35:05 -0800 tron (0.3.1) unstable; urgency=low * Bug fix release * Adding state diagrams to documentation -- Daniel Nephin Tue, 27 Mar 2012 11:35:05 -0800 tron (0.3.0) unstable; urgency=low * !Tags, *references, and &anchors are now deprecated in the trond configuration file. Support will be removed for them in 0.5. * Adding an enabled option for jobs, so they can be configured as disabled by default * tron commands (tronview, tronfig, tronctl) now support a global config (defaults to /etc/tron/tron.yaml) * tronview will now pipe its output through 'less' if appropriate -- Daniel Nephin Mon, 19 Feb 2012 11:35:05 -0800 tron (0.2.10-1) unstable; urgency=low * ssh_options is actually optional (sjohnson) * Cleanup actions no longer cause jobs using an interval scheduler to stop being scheduled if an action fails (sjohnson) * Failed actions can be skipped, causing dependent actions to run (dnephin) * Tests have been moved from test/ to tests/ (sjohnson) * Everything under tron/ web/ and bin/ should now pass pyflakes -- Daniel Nephin Fri, 17 Feb 2012 11:35:05 -0800 tron (0.2.9-1) unstable; urgency=low * tronweb works and is documented (mowings-iseatz) * Daylight Savings Time behavior is more well-defined (sjohnson) * Jobs that fail after running over their next scheduled time are no longer forgotten (sjohnson) * Reconfiguring syslog no longer requires restarting trond to take effect (jbrown) -- Steve Johnson Mon, 6 Feb 2012 16:26:05 -0800 tron (0.2.8.1-1) unstable; urgency=low * Set a meaningful Formatter when logging to syslog (jbrown) * Included prebuilt man pages in distribution so Sphinx isn't required to have them -- James Brown Mon, 12 Dec 2011 16:26:05 -0800 tron (0.2.8-1) unstable; urgency=low * Now on PyPI (irskep) * New HTML documentation at http://packages.python.org/tron (irskep) * Cleanup actions: run a command after the success or failure of a job (irskep) * Logging to syslog with syslog_address config field (irskep) * "zap" command for services (irskep) * simplejson is no longer a dependency for Python 2.6 and up (irskep) * Fix weekday-specified jobs (mon, tues, ...) running a day late (irskep) * Fix services being allowed in jobs list and causing weird crashes (irskep) * Fix missing import in www.py (irskep) * Better resilience to subtlely bad tronfigs (jbrown) -- Steve Johnson Fri, 25 Nov 2011 23:27:00 -0400 tron (0.2.7-1) unstable; urgency=low * Really fix date parsing (rhettg) * Revert instant service monitor so we wait a while before checking our services (rhettg) * Clean up some logging (rhettg) -- Rhett Garber Wed, 14 Sep 2011 16:23:00 -0700 tron (0.2.6-2) unstable; urgency=low * Fix date parsing -- James Brown Wed, 14 Sep 2011 15:36:30 -0700 tron (0.2.6-1) unstable; urgency=low * Support for functional testing. Fixes #49 (irskep) * Context variables for year, month and day. Fixes #57 (irskep) * Integrate Google App Engine Cron scheduling syntax. Fixes #71 (irskep) * Fix crash during service monitoring because of node connect failures. Fixes #77 (rhettg) * Make action runs explicitly not re-startable. Fixes #78 (rhettg) * Flush and fsync state file. Fixes #74 (rhettg) * Handle node disconnect while waiting for channel to start. Fixes #75 (rhettg) * Replace an aggressive assert with a log message for monitor inconsistency. Fixes #73 (rhettg) * Handle tronview event listing issue with garbage collected entites. Fixes #70 (rhettg) * Prevent SSH stampedes by delaying some node EXEC calls (rhettg) -- Rhett Garber Wed, 14 Sep 2011 14:30:15 -0700 tron (0.2.5-1) unstable; urgency=low * Introduce event collection system (rhettg) * Fix a crash in rebuilding all services under certain reconfig scenarios. Fixes #67 (rhettg) * Fix potential service situation where monitors would stop running after failures (rhettg) * Additional logging around startup failures (rhettg) -- Rhett Garber Wed, 22 June 2011 13:24:00 -0800 tron (0.2.4-1) unstable; urgency=low * Final tronfig fix for stdout/stdin behavior (rhettg) -- Rhett Garber Tue, 12 Apr 2011 10:50:00 -0800 tron (0.2.3-2) unstable; urgency=low * Made tronfig work with non-interactive uploads again. (jbrown) -- James Brown Mon, 11 Apr 2011 22:06:56 -0700 tron (0.2.3-1) unstable; urgency=low * Resolved an issue where tronfig via stdin wouldn't catch all errors. (rhettg) * Provided additional config time validation to catch bad configurations. (rhettg) -- Rhett Garber Thu, 7 Apr 2011 10:50:00 -0800 tron (0.2.2-1) unstable; urgency=low * Resolved an issue where certain service reconfigurations would cause the service to be stuck in the DOWN state (rhettg) * Reworked service to keep consistant instance numbers across restarts and reconfigs (rhettg) -- Rhett Garber Wed, 23 Mar 2011 18:26:00 -0800 tron (0.2.1-1) unstable; urgency=low * Resolve an issue where run_time wasn't set for manually started jobs (rhettg) * Support for multiple arguments to tronctl (for starting things in bulk) (rhettg) * Support for starting a job with a specific run_time (rhettg) * Resolved an issue where services, after a reconfig, wouldn't cause state changes (rhettg) * Updated man pages (rhettg) -- Rhett Garber Wed, 09 Feb 2011 15:20:00 -0800 tron (0.2.0-1) unstable; urgency=low * New services system (rhettg) -- Rhett Garber Mon, 06 Feb 2011 15:15:00 -0800 tron (0.1.10-1) unstable; urgency=low * Remove use of deprecated twisted timeout calls. Fixes #9 (rhettg) * Handle newer versions of twisted (rhettg) * Dynamic column widths in tronview and better overflow (ebaum) * Command now displayed in tronview for an action Fixes #32 (ebaum) * Respect tronview -n option for stdout/stderr output. Fixes #41 (ebaum) * Show warnings option for tronview. Fixes #46 (ebaum) * Suppress headers option for tronview. (ebaum) * Fix an issue where default empty config failed to apply. (rhettg) * Add versioning to both tron module, command and state file (rhettg) * Set umask on daemon to allow proper pid-file control (rhettg) * Fix issue with command context not propogating on live reconfigs. Fixes #53 (rhettg) -- Rhett Garber Fri, 14 Jan 2011 13:10:00 -0800 tron (0.1.9-1) unstable; urgency=low * Fix issue with config changes causing previous job runs to be in an unstable state. #42 (ebaum) -- Rhett Garber Mon, 14 Dec 2010 14:12:00 -0800 tron (0.1.8-1) unstable; urgency=low * Address issue with bad format strings in commands causing untold disasters. #45 (rhettg) -- Rhett Garber Mon, 06 Dec 2010 17:38:00 -0800 tron (0.1.7-1) unstable; urgency=low * Improve log rotation scripts under Debian (jbrown) * Fix an issue where removing a job with a live reconfig caused the job not to actually be removed. #44 (rhettg) * Some logging changes to make debugging issues easier (rhettg) * Some cleanup and better error/delay handling around process control for state writing. (rhettg) -- Rhett Garber Mon, 22 Nov 2010 15:52:00 -0800 tron (0.1.6-1) unstable; urgency=low * Fix issue with live reconfigs causing intervals to be skipped (rhettg) * Added log file re-opening on SIGHUP (fixes #37) (rhettg) * Fix some issues with cmp functions for jobs that caused incorrect reconfigs (#38) (mtytel) * Fix issue with manually starting all_node jobs/services (mtytel) -- Rhett Garber Fri, 15 Oct 2010 15:05:00 -0700 tron (0.1.5-1) unstable; urgency=low * Fixed crash due to config bug where SSH options were sometimes missing (rhettg) * Tweaks to command line interface (rhettg) -- Rhett Garber Wed, 14 Sep 2010 10:14:00 -0700 tron (0.1.4-1) UNRELEASED; urgency=low * Simpler default options and config for trond (rhettg) * Trond daemonizing for proper init.d start/stop behavior (rhettg) * Fixes to reduce state file writing (matthewtytel) * Better pre-validation for tronfig (matthewtytel) * Updates to man pages (matthewtytel) -- Rhett Garber Tue, 7 Sep 2010 16:51:00 -0700 tron (0.1.3-3) UNRELEASED; urgency=low * Use /var/lib/tron/ for a working directory (roguelazer) * Fix bug in Node configuration with services (rhettg) -- James Brown Mon, 2 Sep 2010 11:35:00 -0700 tron (0.1.3-2) UNRELEASED; urgency=low * No longer depend on libyaml -- James Brown Mon, 2 Sep 2010 11:00:46 -0700 tron (0.1.3-1) UNRELEASED; urgency=low * Better debian packaging (roguelazer) * Cleaner configuration (rhettg) * SIGHUP handling for reconfiguration (matthewtytel) * Command Context (environment variables for command execution) (rhettg) * Show job duration, alphabetize job list and direct stdout/stderr access (matthewtytel) -- James Brown Mon, 30 Aug 2010 18:33:00 -0700 tron (0.1.2) UNRELEASED; urgency=low * Services (matthewtytel) * Smarter node pools (run all nodes) (matthewtytel) * Randomized node pool selection (matthewtytel) -- Rhett Garber Thu, 19 Aug 2010 11:05:00 -0700 tron (0.1.1) UNRELEASED; urgency=low * On the fly reconfiguration (matthewtytel) * Saving state (matthewtytel) * job enable/disable (matthewtytel) -- Rhett Garber Thu, 19 Aug 2010 11:05:00 -0700 tron (0.1.0) UNRELEASED; urgency=low * Initial release. (Closes: #XXXXXX) -- Rhett Garber Tue, 23 Mar 2010 07:34:36 -0700 ================================================ FILE: debian/compat ================================================ 10 ================================================ FILE: debian/control ================================================ Source: tron Section: admin Priority: optional Maintainer: Daniel Nephin Build-Depends: debhelper (>= 7), python3.10-dev, libdb5.3-dev, libyaml-dev, libssl-dev, libffi-dev, dh-virtualenv Standards-Version: 3.8.3 Package: tron Architecture: all Homepage: http://github.com/yelp/Tron Depends: bsdutils, python3.10, libdb5.3, libyaml-0-2, ${shlibs:Depends}, ${misc:Depends} Description: Tron is a job scheduling, running and monitoring package. Designed to replace Cron for complex scheduling and dependencies. Provides: Centralized configuration for running jobs across multiple machines Dependencies on jobs and resources Monitoring of jobs ================================================ FILE: debian/copyright ================================================ This package was debianized by Steve Johnson on Sat, 26 Nov 2011 15:13:00 -0400. It was downloaded from http://github.com/yelp/Tron Upstream Author: Rhett Garber Matt Tytel Copyright: Copyright 2010 Yelp License: Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: debian/docs ================================================ LICENSE.txt README.md ================================================ FILE: debian/install ================================================ tron/logging.conf var/lib/tron tronweb/ opt/venvs/tron/ ================================================ FILE: debian/pycompat ================================================ 2 ================================================ FILE: debian/pyversions ================================================ 2.5-2.6 ================================================ FILE: debian/rules ================================================ #!/usr/bin/make -f # -*- makefile -*- DH_VERBOSE := 1 %: dh $@ --with python-virtualenv # do not call `make clean` as part of packaging override_dh_auto_clean: true override_dh_auto_build: true # do not call `make test` as part of packaging override_dh_auto_test: true override_dh_virtualenv: echo $(PIP_INDEX_URL) dh_virtualenv --index-url $(PIP_INDEX_URL) \ --extra-pip-arg --trusted-host=169.254.255.254 \ --extra-pip-arg --only-binary=cryptography \ --python=/usr/bin/python3.10 \ --preinstall cython==0.29.36 \ --preinstall pip==24.3.1 \ --preinstall setuptools==65.5.1 @echo patching k8s client lib for configuration class patch debian/tron/opt/venvs/tron/lib/python3.10/site-packages/kubernetes/client/configuration.py contrib/patch-config-loggers.diff override_dh_installinit: dh_installinit --noscripts ================================================ FILE: debian/tron.conffiles ================================================ /var/lib/tron/logging.conf ================================================ FILE: debian/tron.default ================================================ # Defaults for tron initscript # sourced by /etc/init.d/tron # installed at /etc/default/tron by the maintainer scripts # # This is a POSIX shell fragment # # Additional options that are passed to the Daemon. DAEMON_OPTS="--log-conf /var/lib/tron/logging.conf" LISTEN_HOST="0.0.0.0" LISTEN_PORT="8089" # User the daemon will run as. Needs to have appropriate credentials to SSH into your working nodes. # You should take care in setting permissions appropriately for log and working directories on /var DAEMONUSER="" # Enable this when you have configured tron to your liking. RUN="no" ================================================ FILE: debian/tron.dirs ================================================ var/lib/tron/ var/log/tron/ ================================================ FILE: debian/tron.example ================================================ sample_config.yaml ================================================ FILE: debian/tron.links ================================================ opt/venvs/tron/bin/check_tron_jobs usr/bin/check_tron_jobs.py opt/venvs/tron/bin/tronctl usr/bin/tronctl opt/venvs/tron/bin/trond usr/bin/trond opt/venvs/tron/bin/tronfig usr/bin/tronfig opt/venvs/tron/bin/tronview usr/bin/tronview opt/venvs/tron/bin/generate_tron_tab_completion_cache usr/bin/generate_tron_tab_completion_cache opt/venvs/tron/bin/tronctl_tabcomplete.sh usr/share/bash-completion/completions/tronctl opt/venvs/tron/bin/tronview_tabcomplete.sh usr/share/bash-completion/completions/tronview ================================================ FILE: debian/tron.manpages ================================================ docs/source/man/tronctl.1 docs/source/man/trond.8 docs/source/man/tronview.1 docs/source/man/tronfig.1 ================================================ FILE: debian/tron.postinst ================================================ #!/bin/sh -e # # Post-installation script for tron #DEBHELPER# exit 0 ================================================ FILE: debian/tron.service ================================================ [Unit] Description=trond After=network.target # Attempt restarts indefinitely (If omitted, systemd attempts max 5x within StartLimitIntervalSec) StartLimitIntervalSec=0 StartLimitBurst=0 [Service] User=tron EnvironmentFile=/etc/default/tron ExecStartPre=/bin/bash -c 'if pgrep -x trond >/dev/null; then echo "ERROR: trond process already running" >&2; exit 1; fi' ExecStart=/usr/bin/zk-flock -k 60 tron_master_${CLUSTER_NAME} "/usr/bin/trond --lock-file=${LOCKFILE:-$PIDFILE} --working-dir=${WORKINGDIR} --host ${LISTEN_HOST} --port ${LISTEN_PORT} ${DAEMON_OPTS}" ExecStopPost=/usr/bin/logger -t tron_exit_status "SERVICE_RESULT:${SERVICE_RESULT} EXIT_CODE:${EXIT_CODE} EXIT_STATUS:${EXIT_STATUS}" # This is generally not recommended, but we need to not send SIGKILL to the child trond process and instead let the SIGTERM from zk-flock propagate down KillMode=process TimeoutStopSec=20 Restart=always # Wait between restart attempts RestartSec=10 LimitNOFILE=100000 [Install] WantedBy=multi-user.target ================================================ FILE: debian/tron.upstart ================================================ description "trond" start on filesystem and (started networking) stop on shutdown respawn kill timeout 20 script set -a if [ -f /etc/default/tron ] ; then . /etc/default/tron fi if [ "x$RUN" != "xyes" ]; then log_failure_msg "$NAME disabled, please adjust the configuration to your needs " log_failure_msg "and then set RUN to 'yes' in /etc/default/$NAME to enable it." exit 0 fi exec start-stop-daemon --start -c $DAEMONUSER --exec /usr/bin/trond -- $DAEMON_OPTS end script ================================================ FILE: debian/watch ================================================ # Example watch control file for uscan # Rename this file to "watch" and then you can run the "uscan" command # to check for upstream updates and more. # See uscan(1) for format # Compulsory line, this is a version 3 file version=3 http://githubredir.debian.net/github/Yelp/Tron ================================================ FILE: dev/config/MASTER.yaml ================================================ # Please visit y/tron-development for a guide on how to setup Tron for local development state_persistence: name: "tron_state" table_name: "tmp-tron-state" store_type: "dynamodb" buffer_size: 1 dynamodb_region: us-west-1 eventbus_enabled: True ssh_options: agent: True nodes: - hostname: localhost # Replace this with the path relative to your home dir to use # action_runner: # runner_type: "subprocess" # remote_status_path: "pg/tron/status" # remote_exec_path: "pg/tron/.tox/py310/bin" jobs: testjob0: enabled: true node: localhost schedule: "cron * * * * *" run_limit: 5 actions: zeroth: command: env trigger_downstreams: minutely: "{ymdhm}" cpus: 1 mem: 100 testjob1: enabled: false node: localhost schedule: "cron * * * * *" actions: first: command: "sleep 5" cpus: 1 mem: 100 second: command: "echo 'hello world'" requires: [first] triggered_by: - "MASTER.testjob0.zeroth.minutely.{ymdhm}" trigger_downstreams: minutely: "{ymdhm}" cpus: 1 mem: 100 testjob2: enabled: false node: localhost schedule: "cron * * * * *" actions: first: command: "echo 'goodbye, world'" cpus: 1 mem: 100 triggered_by: - "MASTER.testjob1.second.minutely.{ymdhm}" retrier: node: localhost schedule: "cron 0 0 1 1 *" actions: failing: command: exit 1 retries: 1 retries_delay: 5m ================================================ FILE: dev/config/_manifest.yaml ================================================ MASTER: config/MASTER.yaml ================================================ FILE: dev/logging.conf ================================================ [loggers] keys=root, twisted, tron, tron.serialize.runstate.statemanager, tron.api.www.access, task_processing, tron.mesos.task_output, pymesos [handlers] keys=stdoutHandler, accessHandler, nullHandler [formatters] keys=defaultFormatter, accessFormatter [logger_root] level=WARN handlers=stdoutHandler [logger_twisted] level=WARN handlers=stdoutHandler qualname=twisted propagate=0 [logger_tron] level=DEBUG handlers=stdoutHandler qualname=tron propagate=0 [logger_tron.api.www.access] level=DEBUG handlers=accessHandler qualname=tron.api.www.access propagate=0 [logger_tron.serialize.runstate.statemanager] level=DEBUG handlers=stdoutHandler qualname=tron.serialize.runstate.statemanager propagate=0 [logger_task_processing] level=INFO handlers=stdoutHandler qualname=task_processing propagate=0 [logger_pymesos] level=DEBUG handlers=stdoutHandler qualname=pymesos propagate=0 [logger_tron.mesos.task_output] level=INFO handlers=nullHandler qualname=tron.mesos.task_output propagate=0 [handler_stdoutHandler] class=logging.StreamHandler level=DEBUG formatter=defaultFormatter args=() [handler_nullHandler] class=logging.NullHandler level=DEBUG args=() [handler_accessHandler] class=logging.StreamHandler level=DEBUG formatter=accessFormatter args=() [formatter_defaultFormatter] format=%(asctime)s %(name)s %(levelname)s %(message)s [formatter_accessFormatter] format=%(message)s ================================================ FILE: docs/source/_static/nature.css ================================================ /* * nature.css_t * ~~~~~~~~~~~~ * * Sphinx stylesheet -- nature theme. * * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ @import url("basic.css"); /* -- page layout ----------------------------------------------------------- */ body { font-family: Arial, sans-serif; font-size: 100%; background-color: #111; color: #555; margin: 0; padding: 0; } div.documentwrapper { float: left; width: 100%; } div.bodywrapper { margin: 0 0 0 230px; } hr { border: 1px solid #B1B4B6; } div.document { background-color: #eee; } div.body { background-color: #ffffff; color: #3E4349; padding: 0 30px 30px 30px; font-size: 0.9em; } div.footer { color: #555; width: 100%; padding: 13px 0; text-align: center; font-size: 75%; } div.footer a { color: #444; text-decoration: underline; } div.related { background-color: #C41200; line-height: 32px; color: #fff; text-shadow: 0px 1px 0 #444; font-size: 0.9em; } div.related a { color: #F3F3CC; } div.sphinxsidebar { font-size: 0.75em; line-height: 1.5em; } div.sphinxsidebarwrapper{ padding: 20px 0; } div.sphinxsidebar h3, div.sphinxsidebar h4 { font-family: Arial, sans-serif; color: #222; font-size: 1.2em; font-weight: normal; margin: 0; padding: 5px 10px; background-color: #ddd; text-shadow: 1px 1px 0 white } div.sphinxsidebar h4{ font-size: 1.1em; } div.sphinxsidebar h3 a { color: #444; } div.sphinxsidebar p { color: #888; padding: 5px 20px; } div.sphinxsidebar p.topless { } div.sphinxsidebar ul { margin: 10px 20px; padding: 0; color: #000; } div.sphinxsidebar a { color: #444; } div.sphinxsidebar input { border: 1px solid #ccc; font-family: sans-serif; font-size: 1em; } div.sphinxsidebar input[type=text]{ margin-left: 20px; } /* -- body styles ----------------------------------------------------------- */ a { color: #005B81; text-decoration: none; } a:hover { color: #E32E00; text-decoration: underline; } div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { font-family: Arial, sans-serif; background-color: #BED4EB; font-weight: normal; color: #212224; margin: 30px 0px 10px 0px; padding: 5px 0 5px 10px; text-shadow: 0px 1px 0 white } div.body h1 { border-top: 20px solid white; margin-top: 0; font-size: 200%; } div.body h2 { font-size: 150%; background-color: #C8D5E3; } div.body h3 { font-size: 120%; background-color: #D8DEE3; } div.body h4 { font-size: 110%; background-color: #D8DEE3; } div.body h5 { font-size: 100%; background-color: #D8DEE3; } div.body h6 { font-size: 100%; background-color: #D8DEE3; } a.headerlink { color: #c60f0f; font-size: 0.8em; padding: 0 4px 0 4px; text-decoration: none; } a.headerlink:hover { background-color: #c60f0f; color: white; } div.body p, div.body dd, div.body li { line-height: 1.5em; } div.admonition p.admonition-title + p { display: inline; } div.highlight{ background-color: white; } div.note { background-color: #eee; border: 1px solid #ccc; } div.seealso { background-color: #ffc; border: 1px solid #ff6; } div.topic { background-color: #eee; } div.warning { background-color: #ffe4e4; border: 1px solid #f66; } p.admonition-title { display: inline; } p.admonition-title:after { content: ":"; } pre { padding: 10px; background-color: White; color: #222; line-height: 1.2em; border: 1px solid #C6C9CB; font-size: 1.1em; margin: 1.5em 0 1.5em 0; -webkit-box-shadow: 1px 1px 1px #d8d8d8; -moz-box-shadow: 1px 1px 1px #d8d8d8; } tt { background-color: #ecf0f3; color: #222; /* padding: 1px 2px; */ font-size: 1.1em; font-family: monospace; } .viewcode-back { font-family: Arial, sans-serif; } div.viewcode-block:target { background-color: #f4debf; border-top: 1px solid #ac9; border-bottom: 1px solid #ac9; } ================================================ FILE: docs/source/command_context.rst ================================================ .. _built_in_cc: Built-In Command Context Variables ================================== Tron includes some built in command context variables that can be used in command configuration for actions. These variables can be used in the command of an action, using Python's format syntax (``{}``). Once rendered into the command, they will **not** change. This is especially important for datetime-based context variables. Once a run is constructed, the datetime-based variables are "frozen", and will not change, even if the job is retried, or rerun one week later. For example:: # myservice.yaml myjob: node: localhost actions: myaction1: command: "Hello world! I'm {action} for job {name} running on {node}" The command would get rendered at job runtime to:: Hello world! I'm myaction1 for myservice.myjob running on localhost **shortdate** Run date in ``YYYY-MM-DD`` format. Supports simple arithmetic of the form ``{shortdate+6}`` which returns a date 6 days in the future, ``{shortdate-2}`` which returns a date 2 days before the run date. NOTE: this takes into account the job's configured timezone, if any. **ym, ymd, ymdh, ymdhm** Same as ``shortdate`` but better granularity. Arithmetic works with most granular unit: ``ymdh+1`` is +1 hours, ``ymdhm+1`` is +1 minute. NOTE: this takes into account the job's configured timezone, if any. **year** Current year in ``YYYY`` format. Supports the same arithmetic operations as `shortdate`. For example, ``{year-1}`` would return the year previous to the run date. NOTE: this takes into account the job's configured timezone, if any. **month** Current month in `MM` format. Supports the same arithmetic operations as `shortdate`. For example, ``{month+2}`` would return 2 months in the future. NOTE: this takes into account the job's configured timezone, if any. **day** Current day in `DD` format. Supports the same arithmetic operations as `shortdate`. For example, ``{day+1}`` would return the day after the run date. NOTE: this takes into account the job's configured timezone, if any. **hour** Current hour in `HH` (0-23) format. Supports the same arithmetic operations as `shortdate`. For example, ``{hour+1}`` would return the hour after the run hour (mod 24). NOTE: this takes into account the job's configured timezone, if any. **unixtime** Current timestamp. Supports addition and subtraction of seconds. For example ``{unixtime+20}`` would return the timestamp 20 seconds after the jobs runtime. **daynumber** Current day number as an ordinal (datetime.toordinal()). Supports addition and subtraction of days. For example ``{daynumber-3}`` would be 3 days before the run date. NOTE: this takes into account the job's configured timezone, if any. **name** Name of the job (e.g. ``myservice.myjob``). **actionnname** The name of the action (e.g. ``myaction1``). **node** Hostname of the node the action is being run on (e.g. ``localhost``). **runid** Run ID of the job run (e.g. ``sample_job.23``) **cleanup_job_status** ``SUCCESS`` if all actions have succeeded when the cleanup action runs, ``FAILURE`` otherwise. ``UNKNOWN`` if used in an action other than the cleanup action. **last_success** The last successful run date (defaults to current date if there was no previous successful run). Supports date arithmetic using the form ``{last_success#shortdate-1}``. **manual** ``true`` if the job was run manually. ``false`` otherwise. Manual job runs are those runs launched via the ``tronctl start`` command (as opposed to those launched by the scheduler). This variable is useful changing the behavior when jobs are run manually, like adding more verbose logging:: command: "myjob --verbose={manual}" **namespace** The namespace of the config where the job comes from. Often ``MASTER`` or ``servicename``. Usually matches the name of service where the code runs. For example, if the job name is ``myservice.mycooljob.1.myaction``, ``{namespace}`` would be rendered as ``myservice``. Built In Environment Variables ============================== The following environment variables are also in the process environment. These can be used like a normal linux environment variable using ``$``, like ``$TRON_JOB_NAMESPACE`` will be expanded at runtime and replace by the appropriate string. Note: These are **different** that Tron Context Variables, which are referenced using python style f-strings (``{myvariable}``) and are "rendered" into the command only, and not available as normal environment variables. In all examples here, imagine running tronview like ``tronview myservice.myjob.42.myaction``. The example variables represent aspects of that particular action: **TRON_JOB_NAMESPACE** This is the tron config namespace where the job lives. Example: ``myservice``. **TRON_JOB_NAME** This variable is the top level key in the tron configuration file, like ``myjob``. **TRON_RUN_NUM** This is the job run number. Example: ``42``. **TRON_ACTION** This is the action name of the particular job. Example: ``myaction``. ================================================ FILE: docs/source/conf.py ================================================ # # Tron documentation build configuration file, created by # sphinx-quickstart on Mon Nov 7 18:05:54 2011. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import os import sys from unittest.mock import MagicMock sys.path.insert(0, os.path.abspath("../..")) import tron # noqa class Mock(MagicMock): @classmethod def __getattr__(cls, name): return MagicMock() MOCK_MODULES = ["bsddb3"] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. needs_sphinx = "1.0" # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ["sphinx.ext.autodoc", "sphinx.ext.coverage"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # The suffix of source filenames. source_suffix = ".rst" # The encoding of source files. # source_encoding = 'utf-8-sig' # The master toctree document. master_doc = "index" # General information about the project. project = "Tron" copyright = "2011, Yelp, Inc." # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = tron.__version__ # The full version, including alpha/beta/rc tags. release = tron.__version__ # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = '' # Else, today_fmt is used as the format for a strftime call. # today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all documents. # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). # add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. # show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = "nature" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". # html_title = None # A shorter title for the navigation bar. Default is the same as html_title. # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. # html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. # html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. # html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. # html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. # html_additional_pages = {} # If false, no module index is generated. # html_domain_indices = True # If false, no index is generated. # html_use_index = True # If true, the index is split into individual pages for each letter. # html_split_index = False # If true, links to the reST sources are added to the pages. # html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. # html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. # html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. # html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = "Trondoc" # -- Options for LaTeX output -------------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ( "index", "Tron.tex", "Tron Documentation", "Yelp, Inc.", "manual", ), ] # The name of an image file (relative to this directory) to place at the top of # the title page. # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # latex_use_parts = False # If true, show page references after internal links. # latex_show_pagerefs = False # If true, show URL addresses after external links. # latex_show_urls = False # Documents to append as an appendix to all manuals. # latex_appendices = [] # If false, no module index is generated. # latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ( "man_tronview", "tronview", "tronview documentation", ["Yelp, Inc."], 1, ), ( "man_tronfig", "tronfig", "tronfig documentation", ["Yelp, Inc."], 1, ), ( "man_tronctl", "tronctl", "control Tron jobs", ["Yelp, Inc."], 1, ), ( "man_trond", "trond", "trond documentation", ["Yelp, Inc."], 8, ), ] # If true, show URL addresses after external links. # man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ( "index", "Tron", "Tron Documentation", "Yelp, Inc.", "Tron", "One line description of project.", "Miscellaneous", ), ] # Documents to append as an appendix to all manuals. # texinfo_appendices = [] # If false, no module index is generated. # texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. # texinfo_show_urls = 'footnote' ================================================ FILE: docs/source/config.rst ================================================ Configuration ============= .. _config_syntax: Syntax ------ The Tron configuration file uses YAML syntax. The recommended configuration style requires only strings, decimal values, lists, and dictionaries: the subset of YAML that can be losslessly transformed into JSON. (In fact, your configuration can be entirely JSON, since YAML is mostly a strict superset of JSON.) Past versions of Tron used additional YAML-specific features such as tags, anchors, and aliases. These features still work in version 0.3, but are now deprecated. Basic Example ------------- :: ssh_options: agent: true nodes: - name: local hostname: 'localhost' jobs: "getting_node_info": node: local schedule: "cron */10 * * * *" actions: "uname": command: "uname -a" "cpu_info": command: "cat /proc/cpuinfo" requires: [uname] .. _command_context_variables: Command Context Variables ------------------------- **command** attribute values may contain **command context variables** that are inserted at runtime. The **command context** is populated both by Tron (see :ref:`built_in_cc`) and by the config file (see :ref:`command_context`). For example:: jobs: "command_context_demo": node: local schedule: "1st monday in june" actions: "print_run_id": # prints 'command_context_demo.1' on the first run, # 'command_context_demo.2' on the second, etc. command: "echo {runid}" SSH --- **ssh_options** (optional) Options for SSH connections to Tron nodes. When tron runs a job on a node, it will add some jitter (random delay) to the run, which can be configured with the options below. **agent** (optional, default ``False``) Set to ``True`` if :command:`trond` should use an SSH agent. This requires that ``$SSH_AUTH_SOCK`` exists in the environment and points to the correct socket. **identities** (optional, default ``[]``) List of paths to SSH identity files **known_hosts_file** (optional, default ``None``) The path to an ssh known hosts file **connect_timeout** (optional, default ``30``) Timeout in seconds when establishing an ssh connection **idle_connection_timeout** (optional, default ``3600``) Timeout in seconds that an ssh connection can remain idle after which it is closed **jitter_min_load** (optional, default ``4``) Minimum `load` on a node before any jitter is introduced. See `jitter_load_factor` for a description of how load is calculated **jitter_max_delay** (optional, default ``20``) Maximum number of seconds to add to a run **jitter_load_factor** (optional, default ``1``) Factor used to increment the count of running actions for determining the upper bound of jitter to add (ex. A factor of 2 would increase the upper bound by 2 seconds per running action) Example:: ssh_options: agent: false known_hosts_file: /etc/ssh/known_hosts identities: - /home/batch/.ssh/id_dsa-nopasswd connect_timeout: 30 idle_connection_timeout: 3600 jitter_min_load: 4 jitter_max_delay: 20 jitter_load_factor: 1 .. _time_zone: Time Zone --------- **time_zone** (optional) Local time as observed by the system clock. If your system is obeying a time zone with daylight savings time, then some of your jobs may run early or late on the days bordering each mode. See :ref:`dst_notes` for more information. Example:: time_zone: US/Pacific .. _command_context: Command Context --------------- **command_context** Dictionary of custom :ref:`command context variables `. It is an arbitrary set of key-value pairs. Example:: command_context: PYTHON: /usr/bin/python TMPDIR: /tmp See a list of :ref:`built_in_cc`. Output Stream Directory ----------------------- **output_stream_dir** A path to the directory used to store the stdout/stderr logs from jobs. It defaults to the ``--working_dir`` option passed to :ref:`trond`. Example:: output_stream_dir: "/home/tronuser/output/" .. _config_state: State Persistence ----------------- **state_persistence** Configure how trond should persist its state to disk. By default a `shelve` store is used and saved to `./tron_state` in the working directory. **store_type** Valid options are: **shelve** - uses the `shelve` module and saves to a local file **yaml** - uses `yaml` and saves to a local file (this is not recommend and is provided to be backwards compatible with previous versions of Tron). You will need the appropriate python module for the option you choose. **name** The name of this store. This will be the filename for a **shelve** or **yaml** store. **buffer_size** The number of save calls to buffer before writing the state. Defaults to 1, which is no buffering. Example:: state_persistence: store_type: shelve name: tron_store .. _action_runners: Action Runners -------------- **Note:** this is an experimental feature **action_runner** Action runner configuration allows you to run Job actions through a script which records it's pid. This provides support for a max_runtime option on jobs, and allows you to stop or kill the action from :command:`tronctl`. **runner_type** Valid options are: **none** Run actions without a wrapper. This is the default **subprocess** Run actions with a script which records the pid and runs the action command in a subprocess (on the remote node). This requires that :command:`bin/action_runner.py` and :command:`bin/action_status.py` are available on the remote host. **remote_status_path** Path used to store status files. Defaults to `/tmp`. **remote_exec_path** Directory path which contains :command:`action_runner.py` and :command:`action_status.py` scripts. Example:: action_runner: runner_type: "subprocess" remote_status_path: "/tmp/tron" remote_exec_path: "/usr/local/bin" Nodes ----- **nodes** List of nodes. Each node has the following options: **hostname** (required) The hostname or IP address of the node **name** (optional, defaults to ``hostname``) A name to refer to this node **username** (optional, defaults to current user) The name of the user to connect with **port** (optional, defaults to 22) The port number of the node Example:: nodes: - name: node1 hostname: 'batch1' - hostname: 'batch2' # name is 'batch2' Node Pools ---------- **node_pools** List of node pools, each with a ``name`` and ``nodes`` list. ``name`` defaults to the names of each node joined by underscores. Example:: node_pools: - name: pool nodes: [node1, batch1] - nodes: [batch1, node1] # name is 'batch1_node1' Jobs and Actions ---------------- **jobs** List of jobs for Tron to manage. See :doc:`jobs` for the options available to jobs and their actions. .. _config_logging: Logging ------- As of v0.3.3 Logging is no longer configured in the tron configuration file. Tron uses Python's standard logging and by default uses a rotating log file handler that rotates files each day. The default log directory is ``/var/log/tron/tron.log``. To configure logging pass -l to trond. You can modify the default logging.conf by copying it from tron/logging.conf. See http://docs.python.org/howto/logging.html#configuring-logging Interesting logs ~~~~~~~~~~~~~~~~ Most tron logs are named by using pythons `__file__` which uses the modules name. There are a couple special cases: **twisted** Twisted sends its logs to the `twisted` log **tron.api.www.access** API access logs are sent to this log at the INFO log level. They follow a standard apache combined log format. ================================================ FILE: docs/source/developing.rst ================================================ .. _developing: Contributing to Tron ==================== Tron is an open source project and welcomes contributions from the community. The source and issue tracker are hosted on github at http://github.com/yelp/Tron. Setting Up an Environment ------------------------- Tron works well with `virtualenv `_, which can be setup using `virtualenvwrapper `_:: $ mkvirtualenv tron --distribute --no-site-packages $ pip install -r dev/req_dev.txt ``req_dev.txt`` contains a list of packages required for development, to run the tests, and `Sphinx `_ to build the documentation. Coding Standards ---------------- All code should be `PEP8 `_ compliant, and should pass pyflakes without warnings. All new code should include full test coverage, and bug fixes should include a test which reproduces the reported issue. This documentation must also be kept up to date with any changes in functionality. Running Tron in a Sandbox ------------------------- The source package includes a development logging.conf and a sample configuration file with a few test cases. To run a development instance of Tron create a working directory and start :command:`trond` using the following:: $ make dev Running the Tests ----------------- Run the tests using ``make test``. Contributing ------------ There should be a github issue created prior to all pull requests. Pull requests should be made to the ``Yelp:development`` branch, and should include additions to ``CHANGES.txt`` which describe what has changed. ================================================ FILE: docs/source/generated/modules.rst ================================================ tron ==== .. toctree:: :maxdepth: 4 tron ================================================ FILE: docs/source/generated/tron.actioncommand.rst ================================================ tron.actioncommand module ========================= .. automodule:: tron.actioncommand :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.api.adapter.rst ================================================ tron.api.adapter module ======================= .. automodule:: tron.api.adapter :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.api.async_resource.rst ================================================ tron.api.async\_resource module =============================== .. automodule:: tron.api.async_resource :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.api.auth.rst ================================================ tron.api.auth module ==================== .. automodule:: tron.api.auth :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.api.controller.rst ================================================ tron.api.controller module ========================== .. automodule:: tron.api.controller :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.api.requestargs.rst ================================================ tron.api.requestargs module =========================== .. automodule:: tron.api.requestargs :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.api.resource.rst ================================================ tron.api.resource module ======================== .. automodule:: tron.api.resource :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.api.rst ================================================ tron.api package ================ Submodules ---------- .. toctree:: :maxdepth: 4 tron.api.adapter tron.api.async_resource tron.api.auth tron.api.controller tron.api.requestargs tron.api.resource Module contents --------------- .. automodule:: tron.api :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.command_context.rst ================================================ tron.command\_context module ============================ .. automodule:: tron.command_context :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.commands.authentication.rst ================================================ tron.commands.authentication module =================================== .. automodule:: tron.commands.authentication :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.commands.backfill.rst ================================================ tron.commands.backfill module ============================= .. automodule:: tron.commands.backfill :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.commands.client.rst ================================================ tron.commands.client module =========================== .. automodule:: tron.commands.client :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.commands.cmd_utils.rst ================================================ tron.commands.cmd\_utils module =============================== .. automodule:: tron.commands.cmd_utils :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.commands.display.rst ================================================ tron.commands.display module ============================ .. automodule:: tron.commands.display :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.commands.retry.rst ================================================ tron.commands.retry module ========================== .. automodule:: tron.commands.retry :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.commands.rst ================================================ tron.commands package ===================== Submodules ---------- .. toctree:: :maxdepth: 4 tron.commands.authentication tron.commands.backfill tron.commands.client tron.commands.cmd_utils tron.commands.display tron.commands.retry Module contents --------------- .. automodule:: tron.commands :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.config.config_parse.rst ================================================ tron.config.config\_parse module ================================ .. automodule:: tron.config.config_parse :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.config.config_utils.rst ================================================ tron.config.config\_utils module ================================ .. automodule:: tron.config.config_utils :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.config.manager.rst ================================================ tron.config.manager module ========================== .. automodule:: tron.config.manager :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.config.rst ================================================ tron.config package =================== Submodules ---------- .. toctree:: :maxdepth: 4 tron.config.config_parse tron.config.config_utils tron.config.manager tron.config.schedule_parse tron.config.schema tron.config.static_config Module contents --------------- .. automodule:: tron.config :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.config.schedule_parse.rst ================================================ tron.config.schedule\_parse module ================================== .. automodule:: tron.config.schedule_parse :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.config.schema.rst ================================================ tron.config.schema module ========================= .. automodule:: tron.config.schema :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.config.static_config.rst ================================================ tron.config.static\_config module ================================= .. automodule:: tron.config.static_config :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.core.action.rst ================================================ tron.core.action module ======================= .. automodule:: tron.core.action :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.core.actiongraph.rst ================================================ tron.core.actiongraph module ============================ .. automodule:: tron.core.actiongraph :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.core.actionrun.rst ================================================ tron.core.actionrun module ========================== .. automodule:: tron.core.actionrun :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.core.job.rst ================================================ tron.core.job module ==================== .. automodule:: tron.core.job :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.core.job_collection.rst ================================================ tron.core.job\_collection module ================================ .. automodule:: tron.core.job_collection :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.core.job_scheduler.rst ================================================ tron.core.job\_scheduler module =============================== .. automodule:: tron.core.job_scheduler :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.core.jobgraph.rst ================================================ tron.core.jobgraph module ========================= .. automodule:: tron.core.jobgraph :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.core.jobrun.rst ================================================ tron.core.jobrun module ======================= .. automodule:: tron.core.jobrun :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.core.recovery.rst ================================================ tron.core.recovery module ========================= .. automodule:: tron.core.recovery :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.core.rst ================================================ tron.core package ================= Submodules ---------- .. toctree:: :maxdepth: 4 tron.core.action tron.core.actiongraph tron.core.actionrun tron.core.job tron.core.job_collection tron.core.job_scheduler tron.core.jobgraph tron.core.jobrun tron.core.recovery Module contents --------------- .. automodule:: tron.core :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.eventbus.rst ================================================ tron.eventbus module ==================== .. automodule:: tron.eventbus :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.kubernetes.rst ================================================ tron.kubernetes module ====================== .. automodule:: tron.kubernetes :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.manhole.rst ================================================ tron.manhole module =================== .. automodule:: tron.manhole :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.mcp.rst ================================================ tron.mcp module =============== .. automodule:: tron.mcp :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.mesos.rst ================================================ tron.mesos module ================= .. automodule:: tron.mesos :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.metrics.rst ================================================ tron.metrics module =================== .. automodule:: tron.metrics :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.node.rst ================================================ tron.node module ================ .. automodule:: tron.node :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.prom_metrics.rst ================================================ tron.prom\_metrics module ========================= .. automodule:: tron.prom_metrics :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.rst ================================================ tron package ============ Subpackages ----------- .. toctree:: :maxdepth: 4 tron.api tron.commands tron.config tron.core tron.serialize tron.utils Submodules ---------- .. toctree:: :maxdepth: 4 tron.actioncommand tron.command_context tron.eventbus tron.kubernetes tron.manhole tron.mcp tron.mesos tron.metrics tron.node tron.prom_metrics tron.scheduler tron.ssh tron.trondaemon tron.yaml Module contents --------------- .. automodule:: tron :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.scheduler.rst ================================================ tron.scheduler module ===================== .. automodule:: tron.scheduler :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.serialize.filehandler.rst ================================================ tron.serialize.filehandler module ================================= .. automodule:: tron.serialize.filehandler :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.serialize.rst ================================================ tron.serialize package ====================== Subpackages ----------- .. toctree:: :maxdepth: 4 tron.serialize.runstate Submodules ---------- .. toctree:: :maxdepth: 4 tron.serialize.filehandler Module contents --------------- .. automodule:: tron.serialize :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.serialize.runstate.dynamodb_state_store.rst ================================================ tron.serialize.runstate.dynamodb\_state\_store module ===================================================== .. automodule:: tron.serialize.runstate.dynamodb_state_store :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.serialize.runstate.rst ================================================ tron.serialize.runstate package =============================== Submodules ---------- .. toctree:: :maxdepth: 4 tron.serialize.runstate.dynamodb_state_store tron.serialize.runstate.shelvestore tron.serialize.runstate.statemanager tron.serialize.runstate.yamlstore Module contents --------------- .. automodule:: tron.serialize.runstate :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.serialize.runstate.shelvestore.rst ================================================ tron.serialize.runstate.shelvestore module ========================================== .. automodule:: tron.serialize.runstate.shelvestore :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.serialize.runstate.statemanager.rst ================================================ tron.serialize.runstate.statemanager module =========================================== .. automodule:: tron.serialize.runstate.statemanager :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.serialize.runstate.yamlstore.rst ================================================ tron.serialize.runstate.yamlstore module ======================================== .. automodule:: tron.serialize.runstate.yamlstore :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.ssh.rst ================================================ tron.ssh module =============== .. automodule:: tron.ssh :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.trondaemon.rst ================================================ tron.trondaemon module ====================== .. automodule:: tron.trondaemon :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.utils.collections.rst ================================================ tron.utils.collections module ============================= .. automodule:: tron.utils.collections :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.utils.crontab.rst ================================================ tron.utils.crontab module ========================= .. automodule:: tron.utils.crontab :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.utils.exitcode.rst ================================================ tron.utils.exitcode module ========================== .. automodule:: tron.utils.exitcode :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.utils.logreader.rst ================================================ tron.utils.logreader module =========================== .. automodule:: tron.utils.logreader :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.utils.observer.rst ================================================ tron.utils.observer module ========================== .. automodule:: tron.utils.observer :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.utils.persistable.rst ================================================ tron.utils.persistable module ============================= .. automodule:: tron.utils.persistable :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.utils.proxy.rst ================================================ tron.utils.proxy module ======================= .. automodule:: tron.utils.proxy :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.utils.queue.rst ================================================ tron.utils.queue module ======================= .. automodule:: tron.utils.queue :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.utils.rst ================================================ tron.utils package ================== Submodules ---------- .. toctree:: :maxdepth: 4 tron.utils.collections tron.utils.crontab tron.utils.exitcode tron.utils.logreader tron.utils.observer tron.utils.persistable tron.utils.proxy tron.utils.queue tron.utils.state tron.utils.timeutils tron.utils.trontimespec tron.utils.twistedutils Module contents --------------- .. automodule:: tron.utils :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.utils.state.rst ================================================ tron.utils.state module ======================= .. automodule:: tron.utils.state :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.utils.timeutils.rst ================================================ tron.utils.timeutils module =========================== .. automodule:: tron.utils.timeutils :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.utils.trontimespec.rst ================================================ tron.utils.trontimespec module ============================== .. automodule:: tron.utils.trontimespec :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.utils.twistedutils.rst ================================================ tron.utils.twistedutils module ============================== .. automodule:: tron.utils.twistedutils :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/generated/tron.yaml.rst ================================================ tron.yaml module ================ .. automodule:: tron.yaml :members: :undoc-members: :show-inheritance: ================================================ FILE: docs/source/index.rst ================================================ Tron ==== Tron is a centralized system for managing periodic batch processes across a cluster. If this is your first time using Tron, read :doc:`tutorial` and :doc:`overview` to get a better idea of what it is, how it works, and how to use it. .. note:: Please report bugs in the documentation at `our Github issue tracker `_. Table of Contents ----------------- .. toctree:: :maxdepth: 2 whats-new.rst tutorial.rst overview.rst config.rst jobs.rst command_context.rst tronweb.rst tools.rst developing.rst Generated Docs ~~~~~~~~~~~~~~ .. toctree:: :maxdepth: 1 generated/modules Indices and tables ================== * :ref:`genindex` * :ref:`search` ================================================ FILE: docs/source/jobs.rst ================================================ Jobs ==== A job consists of a name, a node/node pool, a list of actions, a schedule, and an optional cleanup action. They are periodic events that do not interact with other jobs while running. If all actions exit with status 0, the job has succeeded. If any action exists with a nonzero status, the job has failed. Required Fields --------------- Jobs are defined in the form of a dictionary, where the **name** is the key. The Name of the job is used in :command:`tronview` and :command:`tronctl`. Here is an example:: jobs: "foo": "schedule": ... "command": ... "actions": "run_first": "command": ... **node** Reference to the node or pool to run the job in. If a pool, the job is run in a random node in the pool. **schedule** When to run this job. Schedule fields can take multiple forms. See :ref:`job_scheduling`. **actions** List of :ref:`actions `. Optional Fields --------------- **monitoring** (default **{}**) Dictionary of key: value pairs to inform the monitoring framework on how to alert teams for job failures. If you're using PaaSTA, for any monitoring fields not specified for a job, Tron will default to those set in `monitoring.yaml` in your soa-configs. You can see more about this behavior in the `PaaSTA docs`_. .. _PaaSTA docs: https://paasta.readthedocs.io/en/latest/yelpsoa_configs.html#monitoring-yaml **team** Team responsible for the job. Must already be defined in the monitoring framework. **page** (default **False**) Boolean on whether or not an alert for this job is page-worthy. **runbook** Runbook associated with the job. **tip** (default **None**) A short 1-line version of the runbook. **notification_email** A comma-separated string of email destinations. Defaults to the "team" default. **slack_channels** A list of Slack channels to send alerts to. Defaults to the team setting. Set an empty list to specify no Slack notifications. **ticket** (default **False**) A Boolean value to enable ticket creation. **project** (default **None**) A string representing the JIRA project that the ticket should go under. Defaults to the team value. **priority** (default **None**) A JIRA ticket priority to use when creating a ticket. This only makes sense to use when in combination with the ticket parameter set to True. This value should be a string value like '0', '1', '3.14', etc. If not set, the default will be the default_priority setting for the sensu team or the default priority used for the JIRA project. **tags** (default **None**) A list of arbitrary tags that can be used in handlers for different metadata needs. **component** (default **None**) A list of components affected by the event. A good example here would be to include the job that is being affected. **description** (default **None**) Human readable text giving more context on any monitoring events. **check_that_every_day_has_a_successful_run** (default **False**) If **True**, the latest job run each day will be checked to see if it was successful. If **False**, only the latest overall run will be checked to see if it was successful. **page_for_expected_runtime** (default **False**) If **True**, when either a job or an action exceeds its configured ``expected_runtime``, the generated alert will be considered "critical" and will page the user. If **False**, then an alert will not page the user. **queueing** (default **True**) If a job run is still running when the next job run is to be scheduled, add the next run to a queue if this is **True**. Otherwise, cancel the job run. Note that if the scheduler used for this job is not defined to queue overlapping then this setting is ignored. **allow_overlap** (default **False**) If **True** new job runs will start even if the previous run is still running. By default new job runs are either cancelled or queued (see **queuing**). **run_limit** (default **50**) Number of runs which will be stored. Once a Job has more then run_limit runs, the output and state for the oldest run are removed. Failed runs will not be removed. **all_nodes** (default **False**) If **True** run this job on each node in the node pool list. If a node appears more than once in the list, the job will be run on that node once for each appearance. If **False** run this job on a random node from the node pool list. If a node appears more than once in the list, the job will be more likely to run on that node, proportionate to the number of appearances. If **node** is not a node pool, this option has no effect. **cleanup_action** Action to run when either all actions have succeeded or the job has failed. See :ref:`job_cleanup_actions`. **enabled** (default **True**) If **False** the job will not be scheduled to run. **max_runtime** (default **None**) A time interval (ex: "2 hours") that limits the duration of each job run. If the job run is still running after this duration, all of its actions are sent SIGTERM. **time_zone** (default **None**) Time zone used for calculating when a job should run. Defaults to None, which means it will use the default time_zone set in the master config. **expected_runtime** (default **24h**) A time interval (ex: "2 hours") that specifies the maximum expected duration of each job run. Single units like (20m, 1h, 2d) are accepted, but you can't use mixed units like (1h 20m) Monitoring will alert if a job run is still running after this duration. Use max_runtime instead if hard limit is needed. .. _job_actions: Actions ------- Actions consist primarily of a **command**. An action's command is executed as soon as its dependencies (specified by **requires**) are satisfied. So if your job has 10 actions, 1 of which depends on the other 9, then Tron will launch the first 9 actions in parallel and run the last one when all have completed successfully. If any action exits with nonzero status, the job will continue to run any actions which do not depend on the failed action. Required Fields ^^^^^^^^^^^^^^^ Actions are defined as a dictionary, where the Name of the action is the key. The Name is used in :command:`tronview` and :command:`tronctl`. **command** Command to run. Commands are run using ``/bin/sh`` so bash expressions will not work, and could cause the job to fail. Optional Fields ^^^^^^^^^^^^^^^ **requires** List of action names that must complete successfully before this action is run. Actions can only require actions in the same job. **node** Node or node pool to run the action on if different from the rest of the job. **retries** An integer representing how many times Tron is allowed to automatically retry the command. Tron will immediately re-run the command if it fails, and the action will not enter the failed state until retries are exhausted. Defaults to None (0 retries allowed). **retries_delay** A timedelta to wait in between retries. **expected_runtime** (default **24h**) A time interval (ex: "2 hours") that specifies the maximum expected duration of each action run. Monitoring will alert if a action run is still running after this duration. **trigger_downstreams** (bool or dict) Upon successful completion of an action, will emit a trigger for every item in the dictionary. When set to ``true``, a default dict of ``{shortdate: "{shortdate}"}`` is assumed. Emitted triggers will be in form: ``....``. See ``triggered_by`` for more information. **triggered_by** (list) When list is not empty, action will not start until all required triggers have been emitted by upstream actions. Unlike with ``requires`` attribute, dependent actions don't have to belong to the same job. ``triggered_by`` template may contain any pattern allowed in ``command`` attribute. See :ref:`shortdate` for an explanation of shortdate Example: :: triggered_by: - "other_namespace.some_job.action1.shortdate.{shortdate-1}" **trigger_timeout** (default **24h**) How long will action wait for dependencies listed in ``triggered_by`` before failing. Is not included in ``expected_runtime``. If upstream job fails, no trigger event will be emitted and downstream jobs will fail with trigger timeout. Re-running upstream job will emit the trigger upon successful completion and if any downstream job is still waiting - it will proceed normally. Timed out downstream jobs will not be re-started, and you need to use ``tronctl publish`` to trigger it manually. . Example Actions ^^^^^^^^^^^^^^^ :: jobs: "convert_logs": node: node1 schedule: start_time: 04:00:00 actions: "verify_logs_present": command: "ls /var/log/app/log_{shortdate-1}.txt" "convert_logs": command: "convert_logs /var/log/app/log_{shortdate-1}.txt /var/log/app_converted/log_{shortdate-1}.txt" requires: [verify_logs_present] .. _job_scheduling: Scheduling ---------- Tron supports four methods for configuring the schedule of a job. Schedulers support a jitter parameter that allows them to vary their runtime by a random time delta. Daily ^^^^^ Run the job on specific days at a specific time. The time expression is ``HH:MM:SS[ MTWRFSU]``. Short form:: schedule: "daily 04:00:00" Short form with days:: schedule: "daily 04:00:00 MWF" Long form:: schedule: type: "daily" value: "07:00:00 MWF" jitter: "10 min" # Optional Cron ^^^^ Schedule a job using cron syntax. Tron supports predefined schedules, ranges, and lists for each field. It supports the *L* in day of month field only (which schedules the job on the last day of the month). Only one of the day fields (day of month and day of week) can have a value. Short form:: schedule: "cron */5 * * 7,8 *" # Every 5 minutes in July and August :: schedule: "cron 0 3-6 * * *" # Every hour between 3am and 6am Long form:: schedule: # long form type: "cron" value: "30 4 L * *" # The last day of the month at 4:30am Complex ^^^^^^^ More powerful version of the daily scheduler based on the one used by Google App Engine's cron library. To use this scheduler, use a string in this format as the schedule:: ("every"|ordinal) (days) ["of|in" (monthspec)] (["at"] HH:MM) **ordinal** Comma-separated list of ``1st`` and so forth. Use ``every`` if you don't want to limit by day of the month. **days** Comma-separated list of days of the week (for example, ``mon``, ``tuesday``, with both short and long forms being accepted); ``every day`` is equivalent to ``every mon,tue,wed,thu,fri,sat,sun`` **monthspec** Comma-separated list of month names (for example, ``jan``, ``march``, ``sep``). If omitted, implies every month. You can also say ``month`` to mean every month, as in ``1,8th,15,22nd of month 09:00``. **HH:MM** Time of day in 24 hour time. Some examples:: 2nd,third mon,wed,thu of march 17:00 every monday at 09:00 1st monday of sep,oct,nov at 17:00 every day of oct at 00:00 In the config:: schedule: "every monday at 09:00" :: schedule: type: "groc daily" value: "every day 11:22" jitter: "5 min" .. _dst_notes: Notes on Daylight Saving Time ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Some system clocks are configured to track local time and may observe daylight savings time. For example, on November 6, 2011, 1 AM occurred twice. Prior to version 0.2.9, this would cause Tron to schedule a daily midnight job to be run an hour early on November 7, at 11 PM. For some jobs this doesn't matter, but for jobs that depend on the availability of data for a day, it can cause a failure. Similarly, some jobs on March 14, 2011 were scheduled an hour late. To avoid this problem, set the :ref:`time_zone` config variable. For example:: time_zone: US/Pacific If a job is scheduled at a time that occurs twice, such as 1 AM on "fall back", it will be run on the *first* occurrence of that time. If a job is scheduled at a time that does not exists, such as 2 AM on "spring forward", it will be run an hour later in the "new" time, in this case 3 AM. In the "old" time this is 2 AM, so from the perspective of previous jobs, it runs at the correct time. In general, Tron tries to schedule a job as soon as is correct, and no sooner. A job that is schedule for 2:30 AM will not run at 3 AM on "spring forward" because that would be half an hour too soon from a pre-switch perspective (2 AM). .. note:: If you experience unexpected scheduler behavior, `file an issue on Tron's Github page `_. .. _job_cleanup_actions: Cleanup Actions --------------- Cleanup actions run after the job succeeds or fails. They are specified just like regular actions except that there is only one per job and it has no name or requirements list. If your job creates shared resources that should be destroyed after a run regardless of success or failure, such as intermediate files or Amazon Elastic MapReduce job flows, you can use cleanup actions to tear them down. The command context variable ``cleanup_job_status`` is provided to cleanup actions and has a value of ``SUCCESS`` or ``FAILURE`` depending on the job's final state. For example:: - # ... cleanup_action: command: "python -m mrjob.tools.emr.job_flow_pool --terminate MY_POOL" States ------ The following are the possible states for a Job and Job Run. Job States ^^^^^^^^^^ **ENABLED** A run is scheduled and new runs will continue to be scheduled. **DISABLED** No new runs will be scheduled, and scheduled runs will be cancelled. **RUNNING** Job run currently in progress. Job Run States ^^^^^^^^^^^^^^ **SCHE** The run is scheduled for a specific time **RUNN** The run is currently running **SUCC** The run completed successfully **FAIL** The run failed **WAITING** The run has actions that are waiting for dependencies **QUE** The run is queued behind another run(s) and will start when said runs finish **CANC** The run was scheduled, but later cancelled. **UNKWN** The run is in an unknown state. This state could indicate a bug in Tron, or an exceptional situation with the infrastructure that requires manual inspection. Actions for this job may in fact still be running, but Tron cannot reach them. Troubleshooting ^^^^^^^^^^^^^^^ **My job doesn't start even though the trigger are emitted?** Check that both jobs are in the same tron master. A "tron master" refers to a cluster; like tron-norcal-devc, tron-nova-prod, etc. Triggers don't work across tron masters! You can emit the event manually using command line or API **S3 consistency issues** If your downstream job relies on s3 list to process data you may see it triggered before S3 had finished replicating. This was previously masked by log_done continuously polling S3 to determine if upstream finished. See STREAMINT-269 for details. .. _shortdate: **What does shortdate in triggers mean?** There are two concepts of shortdate here. shortdate in triggered_by: this shortdate is technically the run_date, indicating when the tron job runs shortdate in command: this shortdate is used by batch jobs to specify which s3 dir it is writing to or polling. ================================================ FILE: docs/source/man/tronctl.1 ================================================ .TH "TRONCTL" "1" "April 24, 2013" "0.6" "Tron" .SH NAME tronctl \- control Tron jobs and services . .nr rst2man-indent-level 0 . .de1 rstReportMargin \\$1 \\n[an-margin] level \\n[rst2man-indent-level] level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] - \\n[rst2man-indent0] \\n[rst2man-indent1] \\n[rst2man-indent2] .. .de1 INDENT .\" .rstReportMargin pre: . RS \\$1 . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] . nr rst2man-indent-level +1 .\" .rstReportMargin post: .. .de UNINDENT . RE .\" indent \\n[an-margin] .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] .nr rst2man-indent-level -1 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. .\" Man page generated from reStructeredText. . .SH SYNOPSYS .sp \fBtronctl [\-\-server ] [\-\-verbose] \fP .SH DESCRIPTION .sp \fBtronctl\fP is the control interface for Tron. \fBtronctl\fP allows you to enable, disable, start, stop and cancel Tron Jobs and Services. .SH OPTIONS .INDENT 0.0 .TP .B \fB\-\-server=\fP Config file containing the address of the server the tron instance is running on .TP .B \fB\-\-verbose\fP Displays status messages along the way .TP .B \fB\-\-run\-date=\fP For starting a new job, specifies the run date that should be set. Defaults to today. .UNINDENT .SH JOB COMMANDS .INDENT 0.0 .TP .B disable Disables the job. Cancels all scheduled and queued runs. Doesn\(aqt schedule any more. .TP .B enable Enables the job and schedules a new run. .TP .B start Creates a new run of the specified job and runs it immediately. .TP .B start Attempt to start the given job run. A Job run only starts if no other instance is running. If the job has already started, it will attempt to start any actions in the SCH or QUE state. .TP .B start Attempt to start the action run. .TP .B restart Creates a new job run with the same run time as this job. .TP .B cancel Cancels the specified job run or action run. .TP .B success Marks the specified job run or action run as succeeded. This behaves the same as the run actually completing. Dependant actions are run and queued runs start. .TP .B skip Marks the specified action run as skipped. This allows dependant actions to run, but will not publish any downstream triggers. .TP .B skip-and-publish Marks the specified action run as skipped. This allows dependant actions to run and will publish downstream triggers. .TP .B fail Marks the specified job run or action run as failed. This behaves the same as the job actually failing. .TP .B stop Stop an action run .TP .B stop Stop (SIGTERM) a service .TP .B kill Force stop (SIGKILL) a service .TP .B kill Force stop (SIGKILL) an action run .TP .B move Rename a job .TP .B publish Pulish a action run trigger. This command can be used to trigger an action run in waiting state. e.g. \fBtronctl publish yelp-main.transaction_extract_report.run.shortdate.2019-10-31\fP .TP .B discard Discard existing actionrun trigger. e.g. \fBtronctl discard yelp-main.transaction_extract_report.run.shortdate.2019-10-31\fP .TP .B version Print tron client and server versions .UNINDENT .SH SERVICE COMMANDS .INDENT 0.0 .TP .B start Start the service. .TP .B stop Stop the service. .UNINDENT .SH EXAMPLES .sp .nf .ft C $ tronctl start job0 New Job Run job0.2 created $ tronctl start job0.3 Job Run job0.3 now in state RUNN $ tronctl cancel job0.4 Job Run job0.4 now in state CANC $ tronctl fail job0.4 Job Run job0.4 now in state FAIL $ tronctl restart job0.4 Job Run job0.4 now in state RUNN $ tronctl success job0.5 Job Run job0.5 now in state SUCC .ft P .fi .SH BUGS .sp Post bugs to \fI\%http://www.github.com/yelp/tron/issues\fP. .SH SEE ALSO .sp \fBtrond\fP (8), \fBtronfig\fP (1), \fBtronview\fP (1), .SH AUTHOR Yelp, Inc. .SH COPYRIGHT 2011, Yelp, Inc. .\" Generated by docutils manpage writer. .\" . ================================================ FILE: docs/source/man/trond.8 ================================================ .TH "TROND" "8" "April 24, 2013" "0.6" "Tron" .SH NAME trond \- trond documentation . .nr rst2man-indent-level 0 . .de1 rstReportMargin \\$1 \\n[an-margin] level \\n[rst2man-indent-level] level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] - \\n[rst2man-indent0] \\n[rst2man-indent1] \\n[rst2man-indent2] .. .de1 INDENT .\" .rstReportMargin pre: . RS \\$1 . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] . nr rst2man-indent-level +1 .\" .rstReportMargin post: .. .de UNINDENT . RE .\" indent \\n[an-margin] .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] .nr rst2man-indent-level -1 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. .\" Man page generated from reStructeredText. . .SH SYNOPSYS .sp \fBtrond [\-\-working\-dir=] [\-\-verbose] [\-\-debug]\fP .SH DESCRIPTION .sp \fBtrond\fP is the tron daemon that manages all jobs and services. .SH OPTIONS .INDENT 0.0 .TP .B \fB\-\-version\fP show program\(aqs version number and exit .TP .B \fB\-h, \-\-help\fP show this help message and exit .TP .B \fB\-\-working\-dir=WORKING_DIR\fP Directory where tron\(aqs state and output is stored (default /var/lib/tron/) .TP .B \fB\-l LOG_CONF, \-\-log\-conf=LOG_CONF\fP Logging configuration file to setup python logger .TP .B \fB\-c CONFIG_FILE, \-\-config\-file=CONFIG_FILE\fP Configuration file to load (default in working dir) .TP .B \fB\-v, \-\-verbose\fP Verbose logging .TP .B \fB\-\-debug\fP Debug mode, extra error reporting, no daemonizing .TP .B \fB\-\-nodaemon\fP (DEPRECATED in 0.9.4) Indicates we should not fork and daemonize the process (default False) .TP .B \fB\-\-lock\-file=LOCKFILE\fP Where to store the lock file of the executing process (default /var/run/tron.lock) .TP .B \fB\-P LISTEN_PORT, \-\-port=LISTEN_PORT\fP What port to listen on, defaults 8089 .TP .B \fB\-H LISTEN_HOST, \-\-host=LISTEN_HOST\fP What host to listen on defaults to localhost .UNINDENT .SH FILES .INDENT 0.0 .TP .B Working directory The directory where state and saved output of processes are stored. .TP .B Lock file Ensures only one daemon runs at a time. .TP .B Log File trond error log, configured from logging.conf .UNINDENT .SH SIGNALS .INDENT 0.0 .TP .B \fISIGINT\fP Graceful shutdown. Waits for running jobs to complete. .TP .B \fISIGTERM\fP Does some cleanup before shutting down. .TP .B \fISIGHUP\fP Reload the configuration file. .TP .B \fISIGUSR1\fP Will drop into an ipdb debugging prompt. .UNINDENT .SH LOGGING .sp Tron uses Python\(aqs standard logging and by default uses a rotating log file handler that rotates files each day. Logs go to \fB/var/log/tron/tron.log\fP. .sp To configure logging pass \-l to trond. You can modify the default logging.conf by coping it from tron/logging.conf. See \fI\%http://docs.python.org/howto/logging.html#configuring-logging\fP .SH BUGS .sp trond has issues around daylight savings time and may run jobs an hour early at the boundary. .sp Post further bugs to \fI\%http://www.github.com/yelp/tron/issues\fP. .SH SEE ALSO .sp \fBtronctl\fP (1), \fBtronfig\fP (1), \fBtronview\fP (1), .SH AUTHOR Yelp, Inc. .SH COPYRIGHT 2011, Yelp, Inc. .\" Generated by docutils manpage writer. .\" . ================================================ FILE: docs/source/man/tronfig.1 ================================================ .TH "TRONFIG" "1" "April 24, 2013" "0.6" "Tron" .SH NAME tronfig \- tronfig documentation . .nr rst2man-indent-level 0 . .de1 rstReportMargin \\$1 \\n[an-margin] level \\n[rst2man-indent-level] level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] - \\n[rst2man-indent0] \\n[rst2man-indent1] \\n[rst2man-indent2] .. .de1 INDENT .\" .rstReportMargin pre: . RS \\$1 . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] . nr rst2man-indent-level +1 .\" .rstReportMargin post: .. .de UNINDENT . RE .\" indent \\n[an-margin] .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] .nr rst2man-indent-level -1 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. .\" Man page generated from reStructeredText. . .SH SYNOPSYS .sp \fBtronfig [\-\-server server_name ] [\-\-verbose | \-v] [] [\-p] [\-]\fP .SH DESCRIPTION .sp \fBtronfig\fP allows live editing of the Tron configuration. It retrieves the configuration file for local editing, verifies the configuration, and sends it back to the tron server. The configuration is applied immediately. .SH OPTIONS .INDENT 0.0 .TP .B \fB\-\-server \fP The server the tron instance is running on .TP .B \fB\-\-verbose\fP Displays status messages along the way .TP .B \fB\-\-version\fP Displays version string .TP .B \fB\-p\fP Print the configuration .TP .B \fBnamespace\fP The configuration namespace to edit. Defaults to MASTER .TP .B \fB\-\fP Read new config from \fBstdin\fP. .UNINDENT .SH CONFIGURATION .sp By default tron will run with a blank configuration file. The config file is saved to \fB/config/\fP by default. See the full documentation at \fI\%http://tron.readthedocs.io/en/latest/config.html\fP. .SH BUGS .sp Post bugs to \fI\%http://www.github.com/yelp/tron/issues\fP. .SH SEE ALSO .sp \fBtrond\fP (8), \fBtronctl\fP (1), \fBtronview\fP (1), .SH AUTHOR Yelp, Inc. .SH COPYRIGHT 2011, Yelp, Inc. .\" Generated by docutils manpage writer. .\" . ================================================ FILE: docs/source/man/tronview.1 ================================================ .TH "TRONVIEW" "1" "April 24, 2013" "0.6" "Tron" .SH NAME tronview \- tronview documentation . .nr rst2man-indent-level 0 . .de1 rstReportMargin \\$1 \\n[an-margin] level \\n[rst2man-indent-level] level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] - \\n[rst2man-indent0] \\n[rst2man-indent1] \\n[rst2man-indent2] .. .de1 INDENT .\" .rstReportMargin pre: . RS \\$1 . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] . nr rst2man-indent-level +1 .\" .rstReportMargin post: .. .de UNINDENT . RE .\" indent \\n[an-margin] .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] .nr rst2man-indent-level -1 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. .\" Man page generated from reStructeredText. . .SH SYNOPSYS .sp \fBtronview [\-n ] [\-\-server ] [\-\-verbose] [ | | ]\fP .SH DESCRIPTION .sp \fBtronview\fP displays the status of tron scheduled jobs and services. .INDENT 0.0 .TP .B tronview Show all configured jobs and services .TP .B tronview Shows details for a job or service. Ex: .sp .nf .ft C $ tronview my_job .ft P .fi .TP .B tronview Show details for specific run or instance. Ex: .sp .nf .ft C $ tronview my_job.0 .ft P .fi .TP .B tronview Show details for specific action run. Ex: .sp .nf .ft C $ tronview my_job.0.my_action .ft P .fi .UNINDENT .SH OPTIONS .INDENT 0.0 .TP .B \fB\-\-version\fP show program\(aqs version number and exit .TP .B \fB\-h, \-\-help\fP show this help message and exit .TP .B \fB\-v, \-\-verbose\fP Verbose logging .TP .B \fB\-n NUM_DISPLAYS, \-\-numshown=NUM_DISPLAYS\fP The maximum number of job runs or lines of output to display(0 for show all). Does not affect the display of all jobs and the display of actions for given job. .TP .B \fB\-\-server=SERVER\fP Server URL to connect to .TP .B \fB\-c, \-\-color\fP Display in color .TP .B \fB\-\-nocolor\fP Display without color .TP .B \fB\-o, \-\-stdout\fP Solely displays stdout .TP .B \fB\-e, \-\-stderr\fP Solely displays stderr .TP .B \fB\-E\fP list all emitted triggers .TP .B \fB\-s, \-\-save\fP Save server and color options to client config file (~/.tron) .UNINDENT .SH STATES .sp For complete list of states with a diagram of valid transitions see \fI\%http://packages.python.org/tron/jobs.html#states\fP and \fI\%http://packages.python.org/tron/services.html#states\fP .SH BUGS .sp Post bugs to \fI\%http://www.github.com/yelp/tron/issues\fP. .SH SEE ALSO .sp \fBtrond\fP (8), \fBtronctl\fP (1), \fBtronfig\fP (1), .SH AUTHOR Yelp, Inc. .SH COPYRIGHT 2011, Yelp, Inc. .\" Generated by docutils manpage writer. .\" . ================================================ FILE: docs/source/man_tronctl.rst ================================================ .. _tronctl: tronctl ======= Synopsis -------- ``tronctl [--server ] [--verbose] `` Description ----------- **tronctl** is the control interface for Tron. :command:`tronctl` allows you to enable, disable, start, stop and cancel Tron Jobs and Services. Options ------- ``--server=`` Config file containing the address of the server the tron instance is running on ``--verbose`` Displays status messages along the way ``--run-date=`` For starting a new job, specifies the run date that should be set. Defaults to today. ``--start-date=`` For backfills, specifies the starting date of the first job of the backfill. Note that many jobs operate on the previous day's data. ``--end-date=`` For backfills, specifies the final date of the backfill. Defaults to today. Note that many jobs operate on the previous day's data. Job Commands ------------ disable Disables the job. Cancels all scheduled and queued runs. Doesn't schedule any more. enable Enables the job and schedules a new run. start Creates a new run of the specified job and runs it immediately. Tron will use the latest version of the code and tron config available for the new run id. start Attempt to start the given job run. A Job run only starts if no other instance is running. If the job has already started, it will attempt to start any actions in the SCH or QUE state. Tron will use the latest version of the code and tron config available for the new run id. start Attempt to start the action run. Tron will use the latest version of the code and tron config available for the new run id. restart Creates a new job run with the same run time as this job. Tron will use the latest version of the code and tron config available for the new run id. retry Re-run an action within an existing job run. Will use the exact same code version and tron config as the previous run. rerun Creates a new job run with the same run time as this job (same as restart). Tron will use the latest version of the code and tron config available for the new run id. backfill Creates a series of tronctl start commands for a sequence of dates. --start-date must be provided for a backfill. cancel Cancels the specified job run or action run. success Marks the specified job run or action run as succeeded. This behaves the same as the run actually completing. Dependent actions are run and queued runs start. skip Marks the specified action run as skipped. This allows dependent actions to run, but will not publish any downstream triggers. skip-and-publish Marks the specified action run as skipped. This allows dependent actions to run and will publish downstream triggers. fail Marks the specified job run or action run as failed. This behaves the same as the job actually failing. stop Stop an action run kill Force stop (SIGKILL) an action run Examples -------- :: $ tronctl start job0 New Job Run job0.2 created $ tronctl start job0.3 Job Run job0.3 now in state RUNN $ tronctl cancel job0.4 Job Run job0.4 now in state CANC $ tronctl fail job0.4 Job Run job0.4 now in state FAIL $ tronctl restart job0.4 Job Run job0.4 now in state RUNN $ tronctl success job0.5 Job Run job0.5 now in state SUCC $ tronctl retry MASTER.job.5.action1 Retrying ActionRun: MASTER.job.5.action1 Bugs ---- Post bugs to http://www.github.com/yelp/tron/issues. See Also -------- **trond** (8), **tronfig** (1), **tronview** (1), ================================================ FILE: docs/source/man_trond.rst ================================================ .. _trond: trond ===== Synopsis -------- ``trond [--working-dir=] [--verbose] [--debug]`` Description ----------- **trond** is the tron daemon that manages all jobs. Options ------- ``--version`` show program's version number and exit ``-h, --help`` show this help message and exit ``--working-dir=WORKING_DIR`` Directory where tron's state and output is stored (default /var/lib/tron/) ``-l LOG_CONF, --log-conf=LOG_CONF`` Logging configuration file to setup python logger ``-c CONFIG_FILE, --config-file=CONFIG_FILE`` Configuration file to load (default in working dir) ``-v, --verbose`` Verbose logging ``--debug`` Debug mode, extra error reporting, no daemonizing ``--nodaemon`` [DEPRECATED in 0.9.4] Indicates we should not fork and daemonize the process (default False) ``--lock-file=LOCKFILE`` Where to store the lock file of the executing process (default /var/run/tron.lock) ``-P LISTEN_PORT, --port=LISTEN_PORT`` What port to listen on, defaults 8089 ``-H LISTEN_HOST, --host=LISTEN_HOST`` What host to listen on defaults to localhost Files ----- Working directory The directory where state and saved output of processes are stored. Lock file Ensures only one daemon runs at a time. Log File trond error log, configured from logging.conf Signals ------- `SIGINT` Graceful shutdown. Waits for running jobs to complete. `SIGTERM` Does some cleanup before shutting down. `SIGHUP` Reload the configuration file. `SIGUSR1` Will drop into an ipdb debugging prompt. Logging ------- Tron uses Python's standard logging and by default uses a rotating log file handler that rotates files each day. Logs go to ``/var/log/tron/tron.log``. To configure logging pass -l to trond. You can modify the default logging.conf by coping it from tron/logging.conf. See http://docs.python.org/howto/logging.html#configuring-logging Bugs ---- trond has issues around daylight savings time and may run jobs an hour early at the boundary. Post further bugs to http://www.github.com/yelp/tron/issues. See Also -------- **tronctl** (1), **tronfig** (1), **tronview** (1), ================================================ FILE: docs/source/man_tronfig.rst ================================================ .. _tronfig: tronfig ======= Synopsis -------- ``tronfig [--server server_name ] [--verbose | -v] [] [-p] [-]`` Description ----------- **tronfig** allows live editing of the Tron configuration. It retrieves the configuration file for local editing, verifies the configuration, and sends it back to the tron server. The configuration is applied immediately. Options ------- ``--server `` The server the tron instance is running on ``--verbose`` Displays status messages along the way ``--version`` Displays version string ``-p`` Print the configuration ``namespace`` The configuration namespace to edit. Defaults to MASTER ``-`` Read new config from ``stdin``. Configuration ------------- By default tron will run with a blank configuration file. The config file is saved to ``/config/`` by default. See the full documentation at http://tron.readthedocs.io/en/latest/config.html. Bugs ---- Post bugs to http://www.github.com/yelp/tron/issues. See Also -------- **trond** (8), **tronctl** (1), **tronview** (1), ================================================ FILE: docs/source/man_tronview.rst ================================================ .. _tronview: tronview ======== Synopsis -------- ``tronview [-n ] [--server ] [--verbose] [ | | ]`` Description ----------- **tronview** displays the status of tron scheduled jobs. tronview Show all configured jobs tronview Shows details for a job. Ex:: $ tronview my_job tronview Show details for specific run or instance. Ex:: $ tronview my_job.0 tronview Show details for specific action run. Ex:: $ tronview my_job.0.my_action Options ------- ``--version`` show program's version number and exit ``-h, --help`` show this help message and exit ``-v, --verbose`` Verbose logging ``-n NUM_DISPLAYS, --numshown=NUM_DISPLAYS`` The maximum number of job runs or lines of output to display(0 for show all). Does not affect the display of all jobs and the display of actions for given job. ``--server=SERVER`` Server URL to connect to ``-c, --color`` Display in color ``--nocolor`` Display without color ``-o, --stdout`` Solely displays stdout ``-e, --stderr`` Solely displays stderr ``-s, --save`` Save server and color options to client config file (~/.tron) States ---------- For complete list of states with a diagram of valid transitions see http://packages.python.org/tron/jobs.html#states Bugs ---- Post bugs to http://www.github.com/yelp/tron/issues. See Also -------- **trond** (8), **tronctl** (1), **tronfig** (1), ================================================ FILE: docs/source/overview.rst ================================================ Overview ======== Batch process scheduling on a single UNIX machines has historically been managed by :command:`cron` and its derivatives. But if you have many batches, complex dependencies between batches, or many machines, maintaining config files across them may be difficult. Tron solves this problem by centralizing the configuration and scheduling of jobs to a single daemon. The Tron system is split into four commands: :ref:`trond` Daemon responsible for scheduling, running, and saving state. Provides an HTTP interface to tools. :ref:`tronview` View job state and output. :ref:`tronctl` Start, stop, enable, disable, and otherwise control jobs. :ref:`tronfig` Change Tron's configuration while the daemon is still running. The config file uses YAML syntax, and is further described in :doc:`config`. Nodes, Jobs and Actions ----------------------- Tron's orders consist of *jobs*. :doc:`Jobs ` contain :ref:`actions ` which may depend on other actions in the same job and run on a schedule. :command:`trond` is given access (via public key SSH) to one or more *nodes* on which to run jobs. For example, this configuration has two nodes, each of which is responsible for a single job:: nodes: hostname: 'localhost' - name: node1 hostname: 'batch1' - name: node2 hostname: 'batch2' jobs: "job0": node: node1 schedule: "cron * * * * *" actions: "batch1action": command: "sleep 3; echo asdfasdf" "job1": node: node2 schedule: "cron * * * * *" actions: "batch2action": command: "cat big.txt; sleep 10" How the nodes are set up and assigned to jobs is entirely up to you. They may have different operating systems, access to different databases, different privileges for the Tron user, etc. See also: * :doc:`jobs` * :doc:`config` .. _overview_pools: Node Pools ---------- Nodes can be grouped into *pools*. To continue the previous example:: node_pools: - name:pool nodes: [node1, node2] jobs: # ... "job2": node: pool schedule: "cron * * * * *" actions: "pool_action": command: "ls /; sleep 1" cleanup_action: command: "echo 'all done'" ``job2``'s action will be run on a random node from ``pool`` every 5 seconds. When ``pool_action`` is complete, ``cleanup_action`` will run on the same node. For more information, see :doc:`jobs`. Caveats ------- While Tron solves many scheduling-related problems, there are a few things to watch out for. **Tron keeps an SSH connection open for the entire lifespan of a process.** This means that to upgrade :command:`trond`, you have to either wait until no jobs are running, or accept an inconsistent state. This limitation is being worked on, and should be improved in later releases. **Tron is under active development.** This means that some things will change. Whenever possible these changes will be backwards compatible, but in some cases there may be non-backwards compatible changes. **Tron does not support unicode.** Tron is built using `twisted `_ which does not support unicode. ================================================ FILE: docs/source/sample_config.yaml ================================================ # optional and settable from the command line working_dir: './working' # optional ssh_options: agent: true # default False identities: # default [] - "/home/batch/.ssh/id_dsa-nopasswd" command_context: PYTHON: /usr/bin/python TMPDIR: /tmp # required nodes: - name: node1 hostname: 'batch1' username: 'tronuser' - name: node2 hostname: 'batch2' username: 'tronuser' node_pools: - name: pool nodes: [node1, node2] jobs: "job0": node: pool all_nodes: True schedule: start_time: 04:00:00 queueing: False actions: "verify_logs_present": command: > ls /var/log/app/log_{shortdate-1}.txt "convert_logs": command: > convert_logs /var/log/app/log_{shortdate-1}.txt \ /var/log/app_converted/log_{shortdate-1}.txt requires: [verify_logs_present] # this will run when the job succeeds or fails cleanup_action: command: "rm /{TMPDIR}/random_temp_file" "job1": node: node schedule: "every monday at 09:00" queueing: False actions: "actionAlone": command: "cat big.txt; sleep 10" ================================================ FILE: docs/source/tools.rst ================================================ Man Pages ========= .. toctree:: :maxdepth: 2 man_tronctl.rst man_trond.rst man_tronfig.rst man_tronview.rst ================================================ FILE: docs/source/tron.yaml ================================================ ssh_options: agent: true nodes: - name: node0 hostname: 'localhost' jobs: "uptime_job": node: node0 schedule: "cron */10 * * * *" actions: "uptimer": command: "uptime" ================================================ FILE: docs/source/tronweb.rst ================================================ .. _tronweb: tronweb ======== tronweb is the web-based UI for tron. See http://localhost:8089/web/ ================================================ FILE: docs/source/tutorial.rst ================================================ Tutorial ======== To install Tron you will need: * A copy of the most recent Tron release from either `github `_ or `pypi `_ (see :ref:`installing_tron`). * A server on which to run :command:`trond`. * One or more batch boxes which will run the Jobs. * An SSH key and a user that will allow the tron daemon to login to all of the batch machines without a password prompt. .. _installing_tron: Installing Tron --------------- The easiest way to install Tron is from PyPI:: $ sudo pip install tron You can also get a copy of the current development release from `github `_. See `setup.py` in the source package for a full list of required packages. If you are interested in working on Tron development see :ref:`developing` for additional requirements and setting up a dev environment. Running Tron ------------- Tron runs as a single daemon, :command:`trond`. On your management node, run:: $ sudo -u trond The chosen user will need SSH access to all your worker nodes, as well as permission to write to the working directory, log file, and lock file (see ``trond --help`` for defaults). You can change these directories using command line options. Also see :ref:`config_logging` on how to change the default logging settings. Once :command:`trond` is running, you can view its status using :command:`tronview` (by default tronview will connect to localhost, use ``--server=: -s`` to specify a different server, and have that setting saved in ``~/.tron``):: $ tronview Jobs: No jobs Configuring Tron ---------------- There are a few options on how to configure tron, but the most straightforward is through tronfig:: $ tronfig This will open your configured :envvar:`$EDITOR` with the current configuration file. Edit your file to be something like this:: ssh_options: agent: true nodes: - name: local hostname: 'localhost' jobs: "getting_node_info": node: local schedule: "cron */10 * * * *" actions: "uname": command: "uname -a" "cpu_info": command: "cat /proc/cpuinfo" requires: [uname] After you exit your editor, the configuration will be validated and uploaded to `trond`. Now if you run :command:`tronview` again, you'll see ``getting_node_info`` as a configured job. Note that it is configured to run 10 minutes from now. This should give you time to examine the job to ensure you really want to run it. :: Jobs: Name State Scheduler Last Success getting_node_info ENABLED INTERVAL:0:10:00 None You can quickly disable a job by using :command:`tronctl`:: $ tronctl disable getting_node_info Job getting_node_info is disabled This will stop scheduled jobs and prevent anymore from being scheduled. You are now in manual control. To manually execute a job immediately, do this:: $ tronctl start getting_node_info New job getting_node_info.1 created You can monitor this job run by using :command:`tronview`:: $ tronview getting_node_info.1 Job Run: getting_node_info.1 State: SUCC Node: localhost Action ID & Command State Start Time End Time Duration .uname SUCC 2011-02-28 16:57:48 2011-02-28 16:57:48 0:00:00 .cpu_info SUCC 2011-02-28 16:57:48 2011-02-28 16:57:48 0:00:00 $ tronview getting_node_info.1.uname Action Run: getting_node_info.1.uname State: SUCC Node: localhost uname -a Requirements: Stdout: Linux dev05 2.6.24-24-server #1 SMP Wed Apr 15 15:41:09 UTC 2009 x86_64 GNU/Linux Stderr: Tron also provides a simple, optional web UI that can be used to get tronview data in a browser. See :doc:`tronweb` for setup instructions. That's it for the basics. You might want to look at :doc:`overview` for a more comprehensive description of how Tron works. ================================================ FILE: docs/source/whats-new.rst ================================================ What's New ========== See the `CHANGELOG `_. ================================================ FILE: itest.sh ================================================ #!/bin/bash set -euxo pipefail export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y software-properties-common gdebi-core curl add-apt-repository -y ppa:deadsnakes/ppa apt-get update gdebi --non-interactive /work/dist/*.deb # TODO: change default MASTER config to not require ssh agent apt-get install -y ssh service ssh start eval $(ssh-agent) trond --help tronfig --help /opt/venvs/tron/bin/python - </dev/null; then break fi if [ "$i" == "5" ]; then echo "Failed to start" kill -9 $TRON_PID exit 1 fi sleep 1 done kill -0 $TRON_PID curl localhost:8089/api/status | grep -qi alive tronfig -p MASTER tronfig -n MASTER /work/testfiles/MASTER.yaml tronfig /work/testfiles/MASTER.yaml cat /work/testfiles/MASTER.yaml | tronfig -n MASTER - if test -L /opt/venvs/tron/lib/python3.10/encodings/punycode.py; then echo "Whoa, the tron package shouldn't have an encoding symlink!" echo "Check out https://github.com/spotify/dh-virtualenv/issues/272" exit 1 fi kill -SIGTERM $TRON_PID wait $TRON_PID || true ================================================ FILE: mypy.ini ================================================ [mypy] python_version = 3.10 # TODO: we'd like to be as strict as we are internally, but we need to fully type Tron first # disallow_any_generics = true disallow_incomplete_defs = True # disallow_untyped_calls = true disallow_untyped_decorators = True # disallow_untyped_defs = true show_column_numbers = True show_error_codes = True show_error_context = True warn_incomplete_stub = True warn_redundant_casts = True warn_return_any = True warn_unreachable = True warn_unused_ignores = True exclude = .tox/ [mypy-clusterman_metrics.*] ignore_missing_imports = True [mypy-twisted.internet.*] ignore_missing_imports = True ================================================ FILE: osx-bdb.sh ================================================ #!/bin/bash export BERKELEYDB_DIR=$(brew --prefix berkeley-db) export YES_I_HAVE_THE_RIGHT_TO_USE_THIS_BERKELEY_DB_VERSION=1 ================================================ FILE: package.json ================================================ { "private": true, "homepage": "./", "dependencies": {}, "browserslist": { "production": [ ">0.2%", "not dead", "not op_mini all" ], "development": [ "last 1 chrome version", "last 1 firefox version", "last 1 safari version" ] }, "devDependencies": { "eslint": "^6.6.0", "eslint-config-airbnb": "^18.2.0", "eslint-plugin-import": "^2.22.0", "eslint-plugin-jsx-a11y": "^6.3.1", "eslint-plugin-react": "^7.20.6", "eslint-plugin-react-hooks": "^4.1.0" }, "resolutions": { "axe-core": "4.7.0" }, "engines": { "node-version-shim": "10.x", "node": ">=10" } } ================================================ FILE: pyproject.toml ================================================ [tool.black] line-length = 120 target_version = ['py310'] ================================================ FILE: requirements-dev-minimal.txt ================================================ asynctest botocore-stubs debugpy flake8 moto mypy pre-commit pytest pytest-asyncio requirements-tools types-boto3 types-cachetools types-psutil types-pytz types-PyYAML types-requests<2.31.0.7 # newer types-requests requires urllib3>=2 types-setuptools ================================================ FILE: requirements-dev.txt ================================================ asynctest==0.12.0 boto3-stubs==1.35.63 botocore-stubs==1.38.19 cfgv==2.0.1 debugpy==1.8.1 distlib==0.3.6 filelock==3.4.1 flake8==5.0.4 identify==2.5.5 iniconfig==1.1.1 mccabe==0.7.0 moto==4.1.0 mypy==1.9.0 mypy-extensions==1.0.0 nodeenv==1.8.0 packaging==19.2 platformdirs==2.5.2 pluggy==0.13.0 pre-commit==2.20.0 py==1.10.0 pycodestyle==2.9.0 pyflakes==2.5.0 pyparsing==2.4.2 pytest==7.0.1 pytest-asyncio==0.14.0 requirements-tools==2.1.0 responses==0.13.0 toml==0.10.2 tomli==2.0.1 types-awscrt==0.27.2 types-boto3==1.0.2 types-cachetools==5.5.0.20240820 types-psutil==6.1.0.20241221 types-pytz==2024.2.0.20240913 types-PyYAML==6.0.12 types-requests==2.31.0.5 types-s3transfer==0.13.0 types-setuptools==75.8.0.20250110 types-urllib3==1.26.25.14 virtualenv==20.17.1 xmltodict==0.12.0 ================================================ FILE: requirements-docs.txt ================================================ Jinja2==3.1.2 markupsafe==2.1.1 mock==3.0.5 Pygments==2.13.0 Sphinx==6.1.3 Sphinx-PyPI-upload==0.2.1 ================================================ FILE: requirements-minimal.txt ================================================ addict # not sure why check-requirements is not picking this up from task_processing[mesos_executor] argcomplete boto3 bsddb3 cryptography dataclasses ecdsa>=0.13.3 http-parser # not sure why check-requirements is not picking this up from task_processing[mesos_executor] humanize ipdb ipython Jinja2>=3.1.2 lockfile prometheus-client psutil py-bcrypt pyasn1 pyformance pymesos # not sure why check-requirements is not picking this up from task_processing[mesos_executor] pyopenssl # vault-tools dependency, but mypy is picking up some code (that we don't use) that imports pyopenssl in Twisted here we are. we could add Twisted[tls], but while hacky - this feels a little more explicit pysensu-yelp PyStaticConfiguration pytimeparse pytz PyYAML>=5.1 requests task_processing[mesos_executor,k8s]>=1.2.0 Twisted>=19.7.0 urllib3>=1.24.2 Werkzeug>=0.15.3 ================================================ FILE: requirements.txt ================================================ addict==2.2.1 argcomplete==1.9.5 asttokens==2.2.1 attrs==19.3.0 Automat==20.2.0 backcall==0.1.0 boto3==1.34.80 botocore==1.34.80 bsddb3==6.2.7 cachetools==4.2.1 certifi==2022.12.7 cffi==1.15.0 charset-normalizer==2.0.12 constantly==15.1.0 cryptography==41.0.5 dataclasses==0.6 decorator==4.4.0 ecdsa==0.13.3 executing==1.2.0 google-auth==1.23.0 http-parser==0.9.0 humanize==4.10.0 hyperlink==19.0.0 idna==2.8 incremental==22.10.0 ipdb==0.13.2 ipython==8.10.0 ipython-genutils==0.2.0 jedi==0.16.0 Jinja2==3.1.2 jmespath==0.9.4 kubernetes==26.1.0 lockfile==0.12.2 MarkupSafe==2.1.1 matplotlib-inline==0.1.3 oauthlib==3.1.0 parso==0.7.0 pexpect==4.7.0 pickleshare==0.7.5 prometheus-client==0.21.1 prompt-toolkit==3.0.38 psutil==5.6.6 ptyprocess==0.6.0 pure-eval==0.2.2 py-bcrypt==0.4 pyasn1==0.4.7 pyasn1-modules==0.2.8 pycparser==2.19 pyformance==0.4 Pygments==2.13.0 pymesos==0.3.9 pyopenssl==24.2.1 pyrsistent==0.15.4 pysensu-yelp==1.0.3 PyStaticConfiguration==0.11.1 python-dateutil==2.8.1 pytimeparse==1.1.8 pytz==2019.3 PyYAML==6.0.1 requests==2.27.1 requests-oauthlib==1.2.0 rsa==4.9 s3transfer==0.10.1 setuptools==65.5.1 six==1.15.0 stack-data==0.6.2 task-processing==1.3.5 traitlets==5.0.0 Twisted==22.10.0 typing-extensions==4.5.0 urllib3==1.25.10 wcwidth==0.1.7 websocket-client==0.56.0 Werkzeug==2.2.3 zope.interface==7.2 ================================================ FILE: setup.cfg ================================================ [build_docs] source-dir = docs/ build-dir = docs/_build all_files = 1 [upload_docs] upload_dir = docs/_build/html ================================================ FILE: setup.py ================================================ try: from setuptools import setup, find_packages assert setup except ImportError: from distutils.core import setup import glob import tron setup( name="tron", version=tron.__version__, provides=["tron"], author="Yelp", author_email="yelplabs@yelp.com", url="http://github.com/Yelp/Tron", description="Job scheduling and monitoring system", classifiers=[ "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Operating System :: OS Independent", "License :: OSI Approved :: Apache Software License", "Topic :: System :: Monitoring", "Topic :: System :: Systems Administration", "Intended Audience :: Developers", "Intended Audience :: System Administrators", "Development Status :: 4 - Beta", ], packages=find_packages( exclude=["tests.*", "tests", "example-cluster"], ) + ["tronweb"], scripts=glob.glob("bin/*") + glob.glob("tron/bin/*.py"), include_package_data=True, long_description=""" Tron is a centralized system for managing periodic batch processes across a cluster. If you find cron or fcron to be insufficient for managing complex work flows across multiple computers, Tron might be for you. For more information, look at the `tutorial `_ or the `full documentation `_. """, ) ================================================ FILE: testfiles/MASTER.yaml ================================================ eventbus_enabled: true state_persistence: name: "/nail/tron/tron_state" store_type: "shelve" buffer_size: 10 ssh_options: agent: False identities: - /work/example-cluster/insecure_key action_runner: runner_type: "subprocess" remote_status_path: "/tmp/tron" remote_exec_path: "/work/tron/bin/" nodes: - hostname: localhost username: root time_zone: US/Eastern jobs: one: node: localhost schedule: "cron */5 * * * *" actions: one: command: exit 1 retries: 3 retries_delay: 1m trigger_downstreams: {ymdhm: "{ymdhm}"} two: node: localhost schedule: "cron */5 * * * *" actions: two: command: sleep 10 && date triggered_by: ["MASTER.one.one.ymdhm.{ymdhm}"] trigger_timeout: 30s ================================================ FILE: testifycompat/__init__.py ================================================ from unittest import TestCase # noqa: F401 from testifycompat.assertions import * # noqa: F401, F403 from testifycompat.fixtures import * # noqa: F401, F403 version = "0.1.2" def run(): raise AssertionError( "Oops, you tried to use testifycompat.run(). This function doesn't " "do anything, it only exists as backwards compatibility with testify. " "You should remove it from your code.", ) ================================================ FILE: testifycompat/assertions.py ================================================ """Compatiblity functions for py.test to migrate code from testify. This is not a complete list, but should hopefully cover most of the common assertions. These assertions should **not** be used in new code, and are only for migrating old tests. """ import pytest def assert_equal(left, right, *args): assert left == right assert_sets_equal = assert_dicts_equal = assert_datetimes_equal = assert_equal assert_equals = assert_equal def assert_true(val): assert val def assert_false(val): assert not val def assert_raises_and_contains(exc, text, func, *args, **kwargs): with pytest.raises(exc) as excinfo: func(*args, **kwargs) text = text if isinstance(text, list) else [text] for item in text: assert item in str(excinfo.exconly()) def assert_raises(exc, func=None, *args, **kwargs): if func is None: return pytest.raises(exc) with pytest.raises(exc): func(*args, **kwargs) def assert_in(item, container): assert item in container def assert_not_in(item, container): assert item not in container def assert_is(left, right): assert left is right def assert_is_not(left, right): assert left is not right def assert_not_equal(left, right): assert left != right def assert_lt(left, right): assert left < right def assert_lte(left, right): assert left <= right def assert_gt(left, right): assert left > right def assert_gte(left, right): assert left >= right def assert_in_range(val, start, end): assert start < val < end def assert_between(val, start, end): assert start <= val <= end def assert_all_in(left, right): """Assert that everything in `left` is also in `right` Note: This is different than `assert_subset()` because python sets use `__hash__()` for comparision whereas `in` uses `__eq__()`. """ for item in left: assert item in right def assert_starts_with(val, prefix): assert val.startswith(prefix) def assert_not_reached(): assert False def assert_empty(iterable): assert len(list(iterable)) == 0 def assert_not_empty(iterable): assert len(list(iterable)) > 0 def assert_length(sequence, expected): assert len(list(sequence)) == expected def assert_sorted_equal(left, right): assert sorted(left) == sorted(right) def assert_isinstance(object_, type_): assert isinstance(object_, type_) ================================================ FILE: testifycompat/bin/__init__.py ================================================ ================================================ FILE: testifycompat/bin/migrate.py ================================================ #!/usr/bin/env python """ .. warning:: This script is still very experimental. Use at your own risk. It will be replaced over time with lib2to3 fixers. Usage: ``python -m testifycompat.bin.migrate `` Example: ``find tests -name *.py | xargs python migrate.py`` """ import functools import re import sys def replace(pattern, repl): return functools.partial(re.sub, pattern, repl) replaces = [ # Replace imports replace(r"^from testify import ", "from testifycompat import "), replace(r"^from testify.assertions import ", "from testifycompat import "), replace(r"^import testify as T", "import testifycompat as T"), # Replace test classes replace( r"^class (?:Test)?(\w+)(?:Test|TestCase)\((?:T\.)?TestCase\):$", "class Test\\1(object):", ), replace( r"^class (?:Test)?(\w+)(?:Test|TestCase)(\(\w+TestCase\)):$", "class Test\\1\\2:", ), # Replace some old assertions replace(r"self.assert_\((.*)\)", "assert \\1"), ] def run_replacement(contents): for line in contents: for replacement in replaces: line = replacement(line) yield line def strip_if_main_run(contents): if len(contents) < 2: return contents if "run()" in contents[-1] and "if __name__ == " in contents[-2]: return contents[:-2] return contents def run_migration_on_file(filename): with open(filename) as fh: lines = fh.read().split("\n") lines = list(run_replacement(lines)) lines = strip_if_main_run(lines) with open(filename, "w") as fh: fh.write("\n".join(lines)) if __name__ == "__main__": for filename in sys.argv[1:]: run_migration_on_file(filename) ================================================ FILE: testifycompat/fixtures.py ================================================ """ Compatibility fixtures for migrating code from testify to py.test .. note:: ``class_`` fixtures must be applied to @classmethods. py.test will not run a class_* fixture that is not attached to a class-level method, so your tests will probably fail. """ import pytest def setup(func): return pytest.fixture(autouse=True)(func) def setup_teardown(func): return pytest.yield_fixture(autouse=True)(func) def teardown(func): def teardown_(*args, **kwargs): yield func(*args, **kwargs) return pytest.yield_fixture(autouse=True)(teardown_) def class_setup(func): return pytest.fixture(autouse=True, scope="class")(func) def class_setup_teardown(func): return pytest.yield_fixture(autouse=True, scope="class")(func) def class_teardown(func): def teardown_(*args, **kwargs): yield func(*args, **kwargs) return pytest.yield_fixture(autouse=True, scope="class")(teardown_) def suite(name, reason=None): """Translate a :func:`testify.suite` decorator into the appropriate :mod:`pytest.mark` call. For the disabled suite this results in a skipped test. For other suites it will return a `pytest.mark.` decorator. """ if name == "disabled": return pytest.mark.skipif(True, reason=reason) return getattr(pytest.mark, name) ================================================ FILE: tests/__init__.py ================================================ from twisted.python import log observer = log.PythonLoggingObserver() observer.start() ================================================ FILE: tests/actioncommand_test.py ================================================ import shlex from unittest import mock from testifycompat import assert_equal from testifycompat import assert_not_equal from testifycompat import setup from testifycompat import TestCase from tests.testingutils import autospec_method from tron import actioncommand from tron.actioncommand import ActionCommand from tron.config import schema from tron.serialize import filehandler class TestActionCommand(TestCase): @setup def setup_command(self): self.serializer = mock.create_autospec(filehandler.FileHandleManager) self.serializer.open.return_value = filehandler.NullFileHandle self.ac = ActionCommand("action.1.do", "do", self.serializer) def test_init(self): assert_equal(self.ac.state, ActionCommand.PENDING) def test_init_no_serializer(self): ac = ActionCommand("action.1.do", "do") ac.write_stdout("something") ac.write_stderr("else") assert_equal(ac.stdout, filehandler.NullFileHandle) ac.done() def test_started(self): assert self.ac.started() assert self.ac.start_time is not None assert_equal(self.ac.state, ActionCommand.RUNNING) def test_started_already_started(self): self.ac.started() assert not self.ac.started() def test_exited(self): self.ac.started() assert self.ac.exited(123) assert_equal(self.ac.exit_status, 123) assert self.ac.end_time is not None def test_exited_from_pending(self): assert self.ac.exited(123) assert_equal(self.ac.state, ActionCommand.FAILSTART) def test_exited_bad_state(self): self.ac.started() self.ac.exited(123) assert not self.ac.exited(1) def test_write_stderr_no_fh(self): message = "this is the message" # Test without a stderr self.ac.write_stderr(message) def test_write_stderr(self): message = "this is the message" serializer = mock.create_autospec(filehandler.FileHandleManager) fh = serializer.open.return_value = mock.create_autospec( filehandler.FileHandleWrapper, ) ac = ActionCommand("action.1.do", "do", serializer) ac.write_stderr(message) fh.write.assert_called_with(message) def test_done(self): self.ac.started() self.ac.exited(123) assert self.ac.done() def test_done_bad_state(self): assert not self.ac.done() def test_handle_errback(self): message = "something went wrong" self.ac.handle_errback(message) assert_equal(self.ac.state, ActionCommand.FAILSTART) assert self.ac.end_time def test_is_unknown(self): assert self.ac.is_unknown def test_is_failed(self): assert not self.ac.is_failed def test_is_failed_true(self): self.ac.exit_status = 255 assert self.ac.is_failed def test_is_complete(self): assert not self.ac.is_complete def test_is_complete_true(self): self.ac.machine.state = self.ac.COMPLETE assert self.ac.is_complete, self.ac.machine.state def test_is_done(self): self.ac.machine.state = self.ac.FAILSTART assert self.ac.is_done, self.ac.machine.state self.ac.machine.state = self.ac.COMPLETE assert self.ac.is_done, self.ac.machine.state class TestCreateActionCommandFactoryFromConfig(TestCase): def test_create_default_action_command_no_config(self): config = () factory = actioncommand.create_action_runner_factory_from_config( config, ) assert_equal(type(factory), actioncommand.NoActionRunnerFactory) def test_create_default_action_command(self): config = schema.ConfigActionRunner( schema.ActionRunnerTypes.none.value, None, None, ) factory = actioncommand.create_action_runner_factory_from_config( config, ) assert type(factory) is actioncommand.NoActionRunnerFactory def test_create_action_command_with_simple_runner(self): status_path, exec_path = "/tmp/what", "/remote/bin" config = schema.ConfigActionRunner( schema.ActionRunnerTypes.subprocess.value, status_path, exec_path, ) factory = actioncommand.create_action_runner_factory_from_config( config, ) assert_equal(factory.status_path, status_path) assert_equal(factory.exec_path, exec_path) class TestSubprocessActionRunnerFactory(TestCase): @setup def setup_factory(self): self.status_path = "status_path" self.exec_path = "exec_path" self.factory = actioncommand.SubprocessActionRunnerFactory( self.status_path, self.exec_path, ) def test_from_config(self): config = mock.Mock() runner_factory = actioncommand.SubprocessActionRunnerFactory.from_config( config, ) assert_equal(runner_factory.status_path, config.remote_status_path) assert_equal(runner_factory.exec_path, config.remote_exec_path) def test_create(self): serializer = mock.create_autospec(actioncommand.StringBufferStore) id, command = "id", "do a thing" autospec_method(self.factory.build_command) action_command = self.factory.create(id, command, serializer) assert_equal(action_command.id, id) assert_equal( action_command.command, self.factory.build_command.return_value, ) assert_equal(action_command.stdout, serializer.open.return_value) assert_equal(action_command.stderr, serializer.open.return_value) def test_build_command_complex_quoting(self): id = "id" command = '/bin/foo -c "foo" --foo "bar"' exec_name = "action_runner.py" actual = self.factory.build_command(id, command, exec_name) assert_equal( shlex.split(actual), [ f"{self.exec_path}/{exec_name}", f"{self.status_path}/{id}", command, id, ], ) def test_build_stop_action_command(self): id, command = "id", "do a thing" autospec_method(self.factory.build_command) action_command = self.factory.build_stop_action_command(id, command) assert_equal( action_command.id, f"{id}.{self.factory.build_command.return_value}", ) assert_equal( action_command.command, self.factory.build_command.return_value, ) def test__eq__true(self): first = actioncommand.SubprocessActionRunnerFactory("a", "b") second = actioncommand.SubprocessActionRunnerFactory("a", "b") assert_equal(first, second) def test__eq__false(self): first = actioncommand.SubprocessActionRunnerFactory("a", "b") second = actioncommand.SubprocessActionRunnerFactory("a", "c") assert_not_equal(first, second) assert_not_equal(first, None) assert_not_equal(first, actioncommand.NoActionRunnerFactory) ================================================ FILE: tests/api/__init__.py ================================================ ================================================ FILE: tests/api/adapter_test.py ================================================ import shutil import tempfile from unittest import mock from testifycompat import assert_equal from testifycompat import run from testifycompat import setup from testifycompat import teardown from testifycompat import TestCase from tests import mocks from tests.assertions import assert_length from tron import node from tron import scheduler from tron.api import adapter from tron.api.adapter import ActionRunAdapter from tron.api.adapter import JobRunAdapter from tron.api.adapter import ReprAdapter from tron.api.adapter import RunAdapter from tron.core import actiongraph from tron.core import actionrun from tron.core import job class MockAdapter(ReprAdapter): field_names = ["one", "two"] translated_field_names = ["three", "four"] def get_three(self): return 3 def get_four(self): return 4 class TestReprAdapter(TestCase): @setup def setup_adapter(self): self.original = mock.Mock(one=1, two=2) self.adapter = MockAdapter(self.original) def test__init__(self): assert_equal(self.adapter._obj, self.original) assert_equal(self.adapter.fields, MockAdapter.field_names) def test_get_translation_mapping(self): expected = { "three": self.adapter.get_three, "four": self.adapter.get_four, } assert_equal(self.adapter.translators, expected) def test_get_repr(self): expected = dict(one=1, two=2, three=3, four=4) assert_equal(self.adapter.get_repr(), expected) class SampleClassStub: def __init__(self): self.true_flag = True self.false_flag = False @adapter.toggle_flag("true_flag") def expects_true(self): return "This is true" @adapter.toggle_flag("false_flag") def expects_false(self): return "This is false" class TestToggleFlag(TestCase): @setup def setup_stub(self): self.stub = SampleClassStub() def test_toggle_flag_true(self): assert_equal(self.stub.expects_true(), "This is true") def test_toggle_flag_false(self): assert not self.stub.expects_false() class TestRunAdapter(TestCase): @setup def setup_adapter(self): self.original = mock.Mock() self.adapter = RunAdapter(self.original) def test_get_state(self): assert_equal(self.adapter.get_state(), self.original.state) @mock.patch("tron.api.adapter.NodeAdapter", autospec=True) def test_get_node(self, mock_node_adapter): assert_equal( self.adapter.get_node(), mock_node_adapter.return_value.get_repr.return_value, ) mock_node_adapter.assert_called_with(self.original.node) def test_get_duration(self): self.original.start_time = None assert_equal(self.adapter.get_duration(), "") class TestActionRunAdapter(TestCase): @setup def setup_adapter(self): self.temp_dir = tempfile.mkdtemp() self.action_run = mock.MagicMock() self.job_run = mock.MagicMock() self.adapter = ActionRunAdapter(self.action_run, self.job_run, 4) @teardown def teardown_adapter(self): shutil.rmtree(self.temp_dir) def test__init__(self): assert_equal(self.adapter.max_lines, 4) assert_equal(self.adapter.job_run, self.job_run) assert_equal(self.adapter._obj, self.action_run) def test_get_repr(self): result = self.adapter.get_repr() assert_equal(result["command"], self.action_run.rendered_command) class TestActionRunGraphAdapter(TestCase): @setup def setup_adapter(self): self.ar1 = mock.MagicMock(action_name="a1") self.ar2 = mock.MagicMock(action_name="a2") self.a1 = mock.MagicMock() self.a2 = mock.MagicMock() self.a1.name = "a1" self.a2.name = "a2" self.action_runs = mock.create_autospec( actionrun.ActionRunCollection, action_graph=actiongraph.ActionGraph( { "a1": self.a1, "a2": self.a2, }, {"a1": set(), "a2": {"a1"}}, {"a1": set(), "a2": set()}, ), ) self.adapter = adapter.ActionRunGraphAdapter(self.action_runs) self.action_runs.__iter__.return_value = [self.ar1, self.ar2] def test_get_repr(self): result = self.adapter.get_repr() assert len(result) == 2 assert self.ar1.id == result[0]["id"] assert ["a1"] == result[1]["dependencies"] assert self.ar1.rendered_command == result[0]["command"] assert self.ar1.command_config.command == result[0]["raw_command"] class TestJobRunAdapter(TestCase): @setup def setup_adapter(self): action_runs = mock.MagicMock() action_runs.__iter__.return_value = iter([mock.Mock(), mock.Mock()]) self.job_run = mock.Mock( action_runs=action_runs, action_graph=mocks.MockActionGraph(), ) self.adapter = JobRunAdapter(self.job_run, include_action_runs=True) def test__init__(self): assert self.adapter.include_action_runs def test_get_runs(self): with mock.patch("tron.api.adapter.ActionRunAdapter", autospec=True): assert_length(self.adapter.get_runs(), 2) def test_get_runs_without_action_runs(self): self.adapter.include_action_runs = False assert_equal(self.adapter.get_runs(), None) class TestNodeAdapter(TestCase): @setup def setup_adapter(self): self.node = mock.create_autospec(node.Node) self.adapter = adapter.NodeAdapter(self.node) def test_repr(self): result = self.adapter.get_repr() assert_equal(result["hostname"], self.node.hostname) assert_equal(result["username"], self.node.username) class TestNodePoolAdapter(TestCase): @setup def setup_adapter(self): self.pool = mock.create_autospec(node.NodePool) self.adapter = adapter.NodePoolAdapter(self.pool) @mock.patch("tron.api.adapter.adapt_many", autospec=True) def test_repr(self, mock_many): result = self.adapter.get_repr() assert_equal(result["name"], self.pool.get_name.return_value) mock_many.assert_called_with( adapter.NodeAdapter, self.pool.get_nodes.return_value, ) class TestJobIndexAdapter(TestCase): @setup def setup_adapter(self): self.job = mock.create_autospec(job.Job) self.adapter = adapter.JobIndexAdapter(self.job) def test_repr(self): result = self.adapter.get_repr() self.job.get_runs.assert_called_with() runs = self.job.get_runs.return_value runs.get_newest.assert_called_with() expected = { "name": self.job.get_name.return_value, "actions": [], } assert_equal(result, expected) def test_get_actions(self): action_run = mock.Mock() job_run = self.job.get_runs.return_value.get_newest.return_value job_run.action_runs.__iter__.return_value = [action_run] result = self.adapter.get_actions() expected = { "name": action_run.action_name, "command": action_run.command_config.command, } assert_equal(result, [expected]) def test_get_actions_no_runs(self): self.job.get_runs.return_value.get_newest.return_value = None result = self.adapter.get_actions() assert_equal(result, []) class TestSchedulerAdapter(TestCase): @setup def setup_adapter(self): self.scheduler = mock.create_autospec(scheduler.GeneralScheduler) self.adapter = adapter.SchedulerAdapter(self.scheduler) @mock.patch("tron.api.adapter.scheduler.get_jitter_str", autospec=True) def test_repr(self, mock_get_jitter): result = self.adapter.get_repr() expected = { "type": self.scheduler.get_name.return_value, "value": self.scheduler.get_value.return_value, "jitter": mock_get_jitter.return_value, } assert_equal(result, expected) mock_get_jitter.assert_called_with(self.scheduler.get_jitter()) if __name__ == "__main__": run() ================================================ FILE: tests/api/auth_test.py ================================================ from unittest.mock import MagicMock from unittest.mock import patch import pytest from twisted.web.server import Request from tron.api.auth import AuthorizationFilter from tron.api.auth import AuthorizationOutcome @pytest.fixture def mock_auth_filter(): with patch("tron.api.auth.requests"): yield AuthorizationFilter("http://localhost:31337/whatever", True) def mock_request(path: str, token: str, method: str): res = MagicMock(spec=Request, path=path.encode(), method=method.encode()) res.getHeader.return_value = token return res def test_is_request_authorized(mock_auth_filter): mock_auth_filter.session.post.return_value.json.return_value = { "result": {"allowed": True, "reason": "User allowed"} } assert mock_auth_filter.is_request_authorized( mock_request("/api/jobs/foobar.run.2", "aaa.bbb.ccc", "get") ) == AuthorizationOutcome(True, "User allowed") mock_auth_filter.session.post.assert_called_once_with( url="http://localhost:31337/whatever", json={ "input": { "path": "/api/jobs/foobar.run.2", "backend": "tron", "token": "aaa.bbb.ccc", "method": "get", "service": "foobar", } }, timeout=2, ) def test_is_request_authorized_fail(mock_auth_filter): mock_auth_filter.session.post.side_effect = Exception assert mock_auth_filter.is_request_authorized( mock_request("/allowed", "eee.ddd.fff", "get") ) == AuthorizationOutcome(False, "Auth backend error") def test_is_request_authorized_malformed(mock_auth_filter): mock_auth_filter.session.post.return_value.json.return_value = {"foo": "bar"} assert mock_auth_filter.is_request_authorized( mock_request("/allowed", "eee.ddd.fff", "post") ) == AuthorizationOutcome(False, "Malformed auth response") def test_is_request_authorized_no_enforce(mock_auth_filter): mock_auth_filter.session.post.return_value.json.return_value = { "result": {"allowed": False, "reason": "Missing token"} } with patch.object(mock_auth_filter, "enforce", False): assert mock_auth_filter.is_request_authorized(mock_request("/foobar", "", "post")) == AuthorizationOutcome( True, "Auth dry-run" ) def test_is_request_authorized_disabled(mock_auth_filter): mock_auth_filter.session.post.return_value.json.return_value = { "result": {"allowed": False, "reason": "Missing token"} } with patch.object(mock_auth_filter, "endpoint", None): assert mock_auth_filter.is_request_authorized(mock_request("/buzz", "", "post")) == AuthorizationOutcome( True, "Auth not enabled" ) @pytest.mark.parametrize( "path,expected", ( ("/api/jobs/someservice.instance/110/run", "someservice"), ("/api/jobs/someweirdservice/110/run", "someweirdservice"), ("/api/jobs/", None), ("/api", None), ), ) def test_extract_service_from_path(path, expected): assert AuthorizationFilter._extract_service_from_path(path) == expected ================================================ FILE: tests/api/controller_test.py ================================================ from unittest import mock import pytest from tron import mcp from tron.api import controller from tron.api.controller import ConfigController from tron.api.controller import EventsController from tron.api.controller import InvalidCommandForActionState from tron.api.controller import JobCollectionController from tron.api.controller import UnknownCommandError from tron.config import ConfigError from tron.config import manager from tron.core import actionrun from tron.core import jobrun from tron.core.job_collection import JobCollection from tron.core.job_scheduler import JobScheduler class TestJobCollectionController: @pytest.fixture(autouse=True) def setup_controller(self): self.collection = mock.create_autospec( JobCollection, enable=mock.Mock(), disable=mock.Mock(), ) self.controller = JobCollectionController(self.collection) def test_handle_command_unknown(self): with pytest.raises(UnknownCommandError): self.controller.handle_command("enableall") self.controller.handle_command("disableall") def test_handle_command_move_non_existing_job(self): self.collection.get_names.return_value = [] result = self.controller.handle_command( "move", old_name="old.test", new_name="new.test", ) assert "doesn't exist" in result def test_handle_command_move_to_existing_job(self): self.collection.get_names.return_value = ["old.test", "new.test"] result = self.controller.handle_command( "move", old_name="old.test", new_name="new.test", ) assert "exists already" in result def test_handle_command_move(self): self.collection.get_names.return_value = ["old.test"] result = self.controller.handle_command( "move", old_name="old.test", new_name="new.test", ) assert "Error" not in result class TestActionRunController: @pytest.fixture(autouse=True) def setup_controller(self): self.action_run = mock.create_autospec( actionrun.ActionRun, cancel=mock.Mock(), ) self.job_run = mock.create_autospec(jobrun.JobRun) self.job_run.is_scheduled = False self.controller = controller.ActionRunController( self.action_run, self.job_run, ) self.job_run.action_runs.cleanup_action_run = None def test_handle_command_start_failed(self): self.job_run.is_scheduled = True result = self.controller.handle_command("start") assert not self.action_run.start.mock_calls assert "cannot be started" in result def test_handle_command_recover_failed(self): self.action_run.is_unknown = False result = self.controller.handle_command("recover") assert not self.action_run.recover.mock_calls assert "cannot be recovered" in result def test_handle_command_mapped_command(self): result = self.controller.handle_command("cancel") self.action_run.cancel.assert_called_with() assert "now in state" in result def test_handle_command_mapped_command_failed(self): self.action_run.cancel.return_value = False with pytest.raises(InvalidCommandForActionState): self.controller.handle_command("cancel") def test_handle_termination_not_implemented(self): self.action_run.stop.side_effect = NotImplementedError result = self.controller.handle_termination("stop") assert "Failed to stop" in result def test_handle_termination_success_without_extra_msg(self): self.action_run.kill.return_value = None result = self.controller.handle_termination("kill") assert "Attempting to kill" in result def test_handle_termination_success_with_extra_msg(self): self.action_run.kill.return_value = "Warning Message" result = self.controller.handle_termination("kill") assert "Attempting to kill" in result assert "Warning Message" in result def test_handle_retry_default(self): self.controller.handle_command("retry") self.action_run.retry.assert_called_once_with(original_command=True) def test_handle_retry_new_command(self): self.controller.handle_command("retry", use_latest_command=True) self.action_run.retry.assert_called_once_with(original_command=False) class TestJobRunController: @pytest.fixture(autouse=True) def setup_controller(self): self.job_run = mock.create_autospec( jobrun.JobRun, run_time=mock.Mock(), cancel=mock.Mock(), ) self.job_scheduler = mock.create_autospec(JobScheduler) self.controller = controller.JobRunController( self.job_run, self.job_scheduler, ) def test_handle_command_restart(self): self.controller.handle_command("restart") self.job_scheduler.manual_start.assert_called_with( self.job_run.run_time, ) def test_handle_mapped_command(self): result = self.controller.handle_command("start") self.job_run.start.assert_called_with() assert "now in state" in result def test_handle_mapped_command_failure(self): self.job_run.cancel.return_value = False result = self.controller.handle_command("cancel") self.job_run.cancel.assert_called_with() assert "Failed to cancel" in result class TestJobController: @pytest.fixture(autouse=True) def setup_controller(self): self.job_scheduler = mock.create_autospec(JobScheduler) self.controller = controller.JobController(self.job_scheduler) def test_handle_command_enable(self): self.controller.handle_command("enable") self.job_scheduler.enable.assert_called_with() def test_handle_command_disable(self): self.controller.handle_command("disable") self.job_scheduler.disable.assert_called_with() def test_handle_command_start(self): run_time = mock.Mock() self.controller.handle_command("start", run_time) self.job_scheduler.manual_start.assert_called_with(run_time=run_time) class TestConfigController: @pytest.fixture(autouse=True) def setup_controller(self): self.mcp = mock.create_autospec(mcp.MasterControlProgram) self.manager = mock.create_autospec(manager.ConfigManager) self.mcp.get_config_manager.return_value = self.manager self.controller = ConfigController(self.mcp) def test_get_config_content_new(self): self.manager.__contains__.return_value = False content = self.controller._get_config_content("name") assert content == self.controller.DEFAULT_NAMED_CONFIG assert not self.manager.read_raw_config.call_count def test_get_config_content_old(self): self.manager.__contains__.return_value = True name = "the_name" content = self.controller._get_config_content(name) assert content == self.manager.read_raw_config.return_value self.manager.read_raw_config.assert_called_with(name) def test_read_config(self): self.manager.__contains__.return_value = True name = "MASTER" resp = self.controller.read_config(name) self.manager.read_raw_config.assert_called_with(name) self.manager.get_hash.assert_called_with(name) assert resp["config"] == self.manager.read_raw_config.return_value assert resp["hash"] == self.manager.get_hash.return_value def test_update_config(self): name, content, config_hash = "foo_namespace", mock.Mock(), mock.Mock() self.manager.get_hash.return_value = config_hash assert not self.controller.update_config(name, content, config_hash) self.manager.get_hash.assert_called_with(name) self.manager.write_config.assert_called_with(name, content) self.mcp.reconfigure.assert_called_with(name) def test_update_config_failure(self): name, content, old_content, config_hash = ( "foo_namespace", mock.Mock(), mock.Mock(), mock.Mock(), ) self.manager.get_hash.return_value = config_hash self.manager.write_config.side_effect = [ConfigError("It broke"), None] self.controller.read_config = mock.Mock(return_value={"config": old_content}) error = self.controller.update_config( name, content, config_hash, ) assert error == "It broke" self.manager.write_config.call_args_list = [ (name, content), (name, old_content), ] assert self.mcp.reconfigure.call_count == 1 self.mcp.reconfigure.assert_called_with(name) def test_update_config_hash_mismatch(self): name, content, config_hash = "foo_namespace", mock.Mock(), mock.Mock() error = self.controller.update_config(name, content, config_hash) assert error == "Configuration has changed. Please try again." def test_delete_config(self): name, content, config_hash = "foo_namespace", "", mock.Mock() self.manager.get_hash.return_value = config_hash assert not self.controller.delete_config(name, content, config_hash) self.manager.delete_config.assert_called_with(name) self.mcp.reconfigure.assert_called_with(name) self.manager.get_hash.assert_called_with(name) def test_delete_config_failure(self): name, content, config_hash = "foo_namespace", "", mock.Mock() self.manager.get_hash.return_value = config_hash self.manager.delete_config.side_effect = Exception("some error") error = self.controller.delete_config(name, content, config_hash) assert error self.manager.delete_config.assert_called_with(name) assert not self.mcp.reconfigure.call_count def test_delete_config_hash_mismatch(self): name, content, config_hash = "foo_namespace", "", mock.Mock() error = self.controller.delete_config(name, content, config_hash) assert error == "Configuration has changed. Please try again." def test_delete_config_content_not_empty(self): name, content, config_hash = "foo_namespace", "", mock.Mock() error = self.controller.delete_config(name, content, config_hash) assert error def test_get_namespaces(self): result = self.controller.get_namespaces() self.manager.get_namespaces.assert_called_with() assert result == self.manager.get_namespaces.return_value class TestEventsController: @pytest.fixture(autouse=True) def setup(self): with mock.patch("tron.api.controller.EventBus", autospec=True) as eb: eb.instance = mock.Mock() self.eventbus = eb self.controller = EventsController() yield def test_info(self): self.eventbus.instance = None assert self.controller.info() == dict(error="EventBus disabled") self.eventbus.instance = mock.Mock() assert self.controller.info() == dict(response=self.eventbus.instance.event_log) def test_publish(self): event = mock.Mock() self.eventbus.instance = None self.eventbus.has_event.return_value = True self.eventbus.publish.return_value = False assert self.controller.info() == dict(error="EventBus disabled") assert len(self.eventbus.publish.mock_calls) == 0 self.eventbus.instance = mock.Mock() assert self.controller.publish(event) == dict(response=f"event {event} already published") assert len(self.eventbus.publish.mock_calls) == 0 self.eventbus.has_event.return_value = False assert self.controller.publish(event) == dict(error=f"could not publish {event}") assert len(self.eventbus.publish.mock_calls) == 1 self.eventbus.publish.return_value = True assert self.controller.publish(event) == dict(response="OK") assert len(self.eventbus.publish.mock_calls) == 2 def test_discard(self): event = mock.Mock() self.eventbus.instance = None self.eventbus.discard.return_value = False assert self.controller.info() == dict(error="EventBus disabled") assert len(self.eventbus.discard.mock_calls) == 0 self.eventbus.instance = mock.Mock() assert self.controller.discard(event) == dict(error=f"could not discard {event}") assert len(self.eventbus.discard.mock_calls) == 1 self.eventbus.discard.return_value = True assert self.controller.discard(event) == dict(response="OK") assert len(self.eventbus.discard.mock_calls) == 2 ================================================ FILE: tests/api/requestargs_test.py ================================================ import datetime from unittest.mock import MagicMock from testifycompat import assert_equal from testifycompat import run from testifycompat import setup from testifycompat import TestCase from tron.api.requestargs import get_bool from tron.api.requestargs import get_datetime from tron.api.requestargs import get_integer from tron.api.requestargs import get_string class TestRequestArgs(TestCase): @setup def setup_args(self): self.args = { b"number": [b"123"], b"string": [b"astring"], b"boolean": [b"1"], b"datetime": [b"2012-03-14 15:09:26"], } self.datetime = datetime.datetime(2012, 3, 14, 15, 9, 26) self.request = MagicMock(args=self.args) def _add_arg(self, name, value): name = name.encode() value = value.encode() if name not in self.args: self.args[name] = [] self.args[name].append(value) def test_get_integer_valid_int(self): self._add_arg("number", "5") assert_equal(get_integer(self.request, "number"), 123) def test_get_integer_invalid_int(self): self._add_arg("nan", "beez") assert not get_integer(self.request, "nan") def test_get_integer_missing(self): assert not get_integer(self.request, "missing") def test_get_string(self): self._add_arg("string", "bogus") assert_equal(get_string(self.request, "string"), "astring") def test_get_string_missing(self): assert not get_string(self.request, "missing") def test_get_bool(self): assert get_bool(self.request, "boolean") def test_get_bool_false(self): self._add_arg("false", "0") assert not get_bool(self.request, "false") def test_get_bool_missing(self): assert not get_bool(self.request, "missing") def test_get_datetime_valid(self): assert_equal(get_datetime(self.request, "datetime"), self.datetime) def test_get_datetime_invalid(self): self._add_arg("nope", "2012-333-4") assert not get_datetime(self.request, "nope") def test_get_datetime_missing(self): assert not get_datetime(self.request, "missing") if __name__ == "__main__": run() ================================================ FILE: tests/api/resource_test.py ================================================ """ Test cases for the web services interface to tron """ from unittest import mock from unittest.mock import MagicMock import pytest import staticconf.testing import twisted.web.http import twisted.web.resource import twisted.web.server from twisted.web import http from tests.assertions import assert_call from tests.testingutils import autospec_method from tron import __version__ from tron import mcp from tron import node from tron.api import controller from tron.core import job from tron.core import jobrun from tron.core.job_collection import JobCollection from tron.core.job_scheduler import JobScheduler with mock.patch( "tron.api.async_resource.AsyncResource.bounded", lambda fn: fn, autospec=None, ): with mock.patch( "tron.api.async_resource.AsyncResource.exclusive", lambda fn: fn, autospec=None, ): from tron.api import resource as www REQUEST = twisted.web.server.Request(mock.Mock(), None) REQUEST.childLink = lambda val: "/jobs/%s" % val def build_request(**kwargs): args = {k.encode(): [v.encode()] for k, v in kwargs.items()} return mock.create_autospec(twisted.web.server.Request, args=args) @pytest.fixture def mock_request(): return build_request() @pytest.fixture def mock_respond(): with mock.patch( "tron.api.resource.respond", autospec=True, ) as mock_respond: mock_respond.side_effect = lambda request, response, code=None: response yield mock_respond @pytest.mark.usefixtures("mock_respond") class WWWTestCase: """Patch www.response to not json encode.""" pass @pytest.mark.parametrize( "response,code,expected_code", [ ("a_string", None, 200), ([{"a": "list"}], None, 200), ({"a": "dict"}, None, 200), ({"a": "dict"}, 501, 501), ({"error": "something went wrong"}, None, 500), ], ) def test_respond(response, code, expected_code): request = build_request() www.respond(request, response, code=code) request.setResponseCode.assert_called_once_with(expected_code) class TestHandleCommand: @pytest.fixture def mock_respond(self, mock_respond): # in this test case, we don't want a side effect mock_respond.side_effect = None return mock_respond def test_handle_command_unknown(self, mock_respond): command = "the command" request = build_request(command=command) mock_controller, obj = mock.Mock(), mock.Mock() error = controller.UnknownCommandError() mock_controller.handle_command.side_effect = error www.handle_command(request, mock_controller, obj) mock_controller.handle_command.assert_called_with(command) mock_respond.assert_called_with( request=request, response={"error": f"Unknown command '{command}' for '{obj}'"}, code=http.NOT_IMPLEMENTED, ) def test_handle_command(self, mock_respond): command = "the command" request = build_request(command=command) mock_controller, obj = mock.Mock(), mock.Mock() www.handle_command(request, mock_controller, obj) mock_controller.handle_command.assert_called_with(command) mock_respond.assert_called_with( request=request, response={"result": mock_controller.handle_command.return_value}, ) def test_handle_command_error(self, mock_respond): command = "the command" request = build_request(command=command) mock_controller, obj = mock.Mock(), mock.Mock() error = Exception("uncaught exception") mock_controller.handle_command.side_effect = error www.handle_command(request, mock_controller, obj) mock_controller.handle_command.assert_called_with(command) mock_respond.assert_called_with(request=request, response={"error": mock.ANY}) class TestActionRunResource(WWWTestCase): @pytest.fixture(autouse=True) def setup_resource(self): self.job_run = mock.MagicMock() self.action_run = mock.MagicMock(output_path=["one"]) with mock.patch("tron.config.static_config.load_yaml_file", autospec=True,), mock.patch( "tron.config.static_config.build_configuration_watcher", autospec=True, ): self.resource = www.ActionRunResource(self.action_run, self.job_run) def test_render_GET(self, mock_respond): request = build_request() mock_config = {"logging.max_lines_to_display": 1000} mock_configuration = staticconf.testing.MockConfiguration(mock_config, namespace="tron") with mock_configuration: response = self.resource.render_GET(request) assert response["id"] == self.resource.action_run.id class TestJobrunResource(WWWTestCase): @pytest.fixture(autouse=True) def setup_resource(self): self.job_run = mock.MagicMock() self.job_scheduler = mock.Mock() self.resource = www.JobRunResource(self.job_run, self.job_scheduler) def test_render_GET(self, mock_request): response = self.resource.render_GET(mock_request) assert response["id"] == self.job_run.id class TestApiRootResource(WWWTestCase): @pytest.fixture(autouse=True) def setup_resource(self): self.mcp = mock.create_autospec(mcp.MasterControlProgram) self.resource = www.ApiRootResource(self.mcp) def test__init__(self): expected_children = [ b"jobs", b"config", b"metrics", b"status", b"events", b"prom-metrics", b"", ] assert set(expected_children) == set(self.resource.children) def test_render_GET(self): expected_keys = [ "jobs", "namespaces", ] response = self.resource.render_GET(build_request()) assert set(response.keys()) == set(expected_keys) self.mcp.get_job_collection().get_jobs.assert_called_with() class TestRootResource(WWWTestCase): @pytest.fixture(autouse=True) def setup_resource(self): self.web_path = "/bogus/path" self.mcp = mock.create_autospec(mcp.MasterControlProgram) self.resource = www.RootResource(self.mcp, self.web_path) def test_render_GET(self): request = build_request() response = self.resource.render_GET(request) assert response == 1 assert request.redirect.call_count == 1 request.finish.assert_called_with() def test_get_children(self): assert set(self.resource.children) == {b"api", b"web", b""} class TestActionRunHistoryResource(WWWTestCase): @pytest.fixture(autouse=True) def setup_resource(self): self.action_runs = [mock.MagicMock(), mock.MagicMock()] self.resource = www.ActionRunHistoryResource(self.action_runs) def test_render_GET(self, request): response = self.resource.render_GET(request) assert len(response) == len(self.action_runs) class TestJobCollectionResource(WWWTestCase): @pytest.fixture(autouse=True) def setup_resource(self): job_collection = mock.create_autospec(JobCollection) job_collection.get_by_name = lambda name: name if name == "testname" else None self.resource = www.JobCollectionResource(job_collection) def test_render_GET(self): self.resource.get_data = MagicMock() result = self.resource.render_GET(REQUEST) assert_call(self.resource.get_data, 0, False, False, True, True) assert "jobs" in result def test_getChild(self): child = self.resource.getChild(b"testname", mock.Mock()) assert isinstance(child, www.JobResource) def test_getChild_missing_job(self): child = self.resource.getChild(b"bar", mock.Mock()) assert isinstance(child, www.ErrorResource) class TestJobResource(WWWTestCase): @pytest.fixture(autouse=True) def setup_resource(self): self.job_scheduler = mock.create_autospec(JobScheduler) self.job_runs = mock.create_autospec(jobrun.JobRunCollection) self.job = mock.create_autospec( job.Job, runs=self.job_runs, all_nodes=False, allow_overlap=True, queueing=True, action_graph=mock.MagicMock(), scheduler=mock.Mock(), node_pool=mock.create_autospec(node.NodePool), max_runtime=mock.Mock(), expected_runtime=mock.MagicMock(), ) self.job.get_name.return_value = "foo" self.job_scheduler.get_job.return_value = self.job self.job_scheduler.get_job_runs.return_value = self.job_runs self.resource = www.JobResource(self.job_scheduler) def test_render_GET(self, mock_request): result = self.resource.render_GET(mock_request) assert result["name"] == self.job_scheduler.get_job().get_name() def test_get_run_from_identifier_HEAD(self): job_run = self.resource.get_run_from_identifier("HEAD") self.job_scheduler.get_job_runs.assert_called_with() assert job_run == self.job_runs.get_newest.return_value def test_get_run_from_identifier_number(self): job_run = self.resource.get_run_from_identifier("3") self.job_scheduler.get_job_runs.assert_called_with() assert job_run == self.job_runs.get_run_by_num.return_value self.job_runs.get_run_by_num.assert_called_with(3) def test_get_run_from_identifier_negative_index(self): job_run = self.resource.get_run_from_identifier("-2") assert job_run == self.job_runs.get_run_by_index.return_value self.job_runs.get_run_by_index.assert_called_with(-2) def test_getChild(self): autospec_method(self.resource.get_run_from_identifier) identifier = b"identifier" resource = self.resource.getChild(identifier, None) assert resource.job_run == self.resource.get_run_from_identifier.return_value def test_getChild_action_run_history(self): autospec_method( self.resource.get_run_from_identifier, return_value=None, ) action_name = "action_name" action_runs = [mock.Mock(), mock.Mock()] self.job.action_graph.names.return_value = [action_name] self.job.runs.get_action_runs.return_value = action_runs resource = self.resource.getChild(action_name, None) assert resource.__class__ == www.ActionRunHistoryResource assert resource.action_runs == action_runs class TestConfigResource: @pytest.fixture(autouse=True) def setup_resource(self): self.mcp = mock.create_autospec(mcp.MasterControlProgram) self.resource = www.ConfigResource(self.mcp) self.controller = self.resource.controller = mock.create_autospec( controller.ConfigController, ) def test_render_GET(self, mock_respond): name = "the_name" request = build_request(name=name) self.resource.render_GET(request) self.controller.read_config.assert_called_with(name) mock_respond.assert_called_with( request=request, response=self.resource.controller.read_config.return_value, ) def test_render_POST_update(self, mock_respond): name, config, hash = "the_name", "config", "hash" request = build_request(name=name, config=config, hash=hash) self.resource.render_POST(request) self.resource.controller.update_config.assert_called_with(name, config, hash) expected_response = { "status": "Active", "error": self.resource.controller.update_config.return_value, } mock_respond.assert_called_with(request=request, response=expected_response) def test_render_POST_delete(self, mock_respond): name, config, hash = "the_name", "", "" request = build_request(name=name, config=config, hash=hash) self.resource.render_POST(request) self.resource.controller.delete_config.assert_called_with(name, config, hash) expected_response = { "status": "Active", "error": self.resource.controller.delete_config.return_value, } mock_respond.assert_called_with(request=request, response=expected_response) class TestStatusResource: def test_render_GET(self, request, mock_respond): self.mcp = mock.create_autospec(mcp.MasterControlProgram) self.mcp.boot_time = 999 resource = www.StatusResource(self.mcp) resource.render_GET(request) expected_response = { "status": "I'm alive.", "version": __version__, "boot_time": self.mcp.boot_time, } mock_respond.assert_called_with( request=request, response=expected_response, ) class TestMetricsResource: @mock.patch("tron.api.resource.view_all_metrics", autospec=True) def test_render_GET(self, mock_view_metrics, request, mock_respond): resource = www.MetricsResource() resource.render_GET(request) mock_respond.assert_called_with( request=request, response=mock_view_metrics.return_value, ) class TestTronSite: @mock.patch("tron.api.resource.meter", autospec=True) def test_log_request(self, mock_meter): site = www.TronSite.create( mock.create_autospec(mcp.MasterControlProgram), "webpath", ) request = mock.Mock(code=500) site.log(request) assert mock_meter.call_count == 1 ================================================ FILE: tests/assertions.py ================================================ """ Assertions for testify. """ from testifycompat import assert_in from testifycompat import assert_not_reached def assert_raises(expected_exception_class, callable_obj, *args, **kwargs): """Returns the exception if the callable raises expected_exception_class""" try: callable_obj(*args, **kwargs) except expected_exception_class as e: # we got the expected exception return e assert_not_reached( "No exception was raised (expected %s)" % expected_exception_class, ) def assert_length(sequence, expected, msg=None): """Assert that a sequence or iterable has an expected length.""" msg = msg or "%(sequence)s has length %(length)s expected %(expected)s" length = len(list(sequence)) assert length == expected, msg % locals() def assert_call(mock, call_idx, *args, **kwargs): """Assert that a function was called on mock with the correct args.""" actual = mock.mock_calls[call_idx] if mock.mock_calls else None msg = f"Call {call_idx} expected {(args, kwargs)}, was {actual}" assert actual == (args, kwargs), msg def assert_mock_calls(expected, mock_calls): """Assert that all expected calls are in the list of mock_calls.""" for expected_call in expected: assert_in(expected_call, mock_calls) ================================================ FILE: tests/bin/__init__.py ================================================ ================================================ FILE: tests/bin/action_runner_test.py ================================================ import tempfile from unittest import mock import pytest from testifycompat import assert_equal from testifycompat import setup from testifycompat import setup_teardown from testifycompat import TestCase from tron.bin import action_runner class TestStatusFile(TestCase): @setup def setup_status_file(self): self.filename = tempfile.NamedTemporaryFile().name self.status_file = action_runner.StatusFile(self.filename) def test_get_content(self): command, proc, run_id = "do this", mock.Mock(), "Job.test.1" with ( mock.patch("tron.bin.action_runner.time.time", autospec=True) as faketime, mock.patch("tron.bin.action_runner.os.getpid", autospec=True) as fakepid, ): faketime.return_value = 0 fakepid.return_value = 2 content = self.status_file.get_content( command=command, proc=proc, run_id=run_id, ) expected = dict( run_id=run_id, command=command, pid=proc.pid, return_code=proc.returncode, runner_pid=2, timestamp=0, ) assert_equal(content, expected) class TestRegister(TestCase): mock_isdir = mock_status_file = None mock_makedirs = None @setup_teardown def patch_sys(self): with ( mock.patch("tron.bin.action_runner.os.path.isdir", autospec=True) as self.mock_isdir, mock.patch("tron.bin.action_runner.os.makedirs", autospec=True) as self.mock_makedirs, mock.patch("tron.bin.action_runner.os.access", autospec=True) as self.mock_access, mock.patch("tron.bin.action_runner.StatusFile", autospec=True) as self.mock_status_file, ): self.output_path = "/bogus/path/does/not/exist" self.command = "command" self.run_id = "Job.test.1" self.proc = mock.Mock() self.proc.wait.return_value = 0 yield def test_validate_output_dir_does_not_exist(self): self.mock_isdir.return_value = False self.mock_access.return_value = True action_runner.validate_output_dir(self.output_path) self.mock_makedirs.assert_called_with(self.output_path) def test_validate_output_dir_does_not_exist_create_fails(self): self.mock_isdir.return_value = False self.mock_access.return_value = True self.mock_makedirs.side_effect = OSError with pytest.raises(OSError): action_runner.validate_output_dir(self.output_path) def test_validate_output_dir_exists_not_writable(self): self.mock_isdir.return_value = True self.mock_access.return_value = False with pytest.raises(OSError): action_runner.validate_output_dir(self.output_path) def test_run_proc(self): self.mock_isdir.return_value = True self.mock_access.return_value = True action_runner.run_proc( self.output_path, self.command, self.run_id, self.proc, ) self.mock_status_file.assert_called_with( self.output_path + "/" + action_runner.STATUS_FILE, ) self.mock_status_file.return_value.wrap.assert_called_with( command=self.command, run_id=self.run_id, proc=self.proc, ) self.proc.wait.assert_called_with() class TestBuildEnvironment: def test_build_environment(self): with mock.patch( "tron.bin.action_runner.os.environ", dict(PATH="/usr/bin/nowhere"), autospec=None, ): env = action_runner.build_environment("MASTER.foo.10.bar") assert env == dict( PATH="/usr/bin/nowhere", TRON_JOB_NAMESPACE="MASTER", TRON_JOB_NAME="foo", TRON_RUN_NUM="10", TRON_ACTION="bar", ) def test_build_environment_invalid_run_id(self): with mock.patch( "tron.bin.action_runner.os.environ", dict(PATH="/usr/bin/nowhere"), autospec=None, ): env = action_runner.build_environment("asdf") assert env == dict( PATH="/usr/bin/nowhere", TRON_JOB_NAMESPACE="UNKNOWN", TRON_JOB_NAME="UNKNOWN", TRON_RUN_NUM="UNKNOWN", TRON_ACTION="UNKNOWN", ) def test_build_environment_too_long_run_id(self): with mock.patch( "tron.bin.action_runner.os.environ", dict(PATH="/usr/bin/nowhere"), autospec=None, ): env = action_runner.build_environment("MASTER.foo.10.bar.baz") assert env == dict( PATH="/usr/bin/nowhere", TRON_JOB_NAMESPACE="MASTER", TRON_JOB_NAME="foo", TRON_RUN_NUM="10", TRON_ACTION="bar.baz", ) class TestBuildLabels: def test_build_labels(self): labels = action_runner.build_labels("MASTER.foo.10.bar") assert labels == { "tron.yelp.com/run_num": "10", } def test_build_labels_with_merging(self): current_labels = {"LABEL1": "value_1"} labels = action_runner.build_labels("MASTER.foo.10.bar", current_labels) assert labels == { "tron.yelp.com/run_num": "10", "LABEL1": "value_1", } def test_build_labels_with_merging_on_unknown(self): current_labels = {"LABEL1": "value_1"} labels = action_runner.build_labels("asdf", current_labels) assert labels == { "tron.yelp.com/run_num": "UNKNOWN", "LABEL1": "value_1", } def test_build_labels_invalid_run_id(self): labels = action_runner.build_labels("asdf") assert labels == { "tron.yelp.com/run_num": "UNKNOWN", } def test_build_labels_too_long_run_id(self): labels = action_runner.build_labels("MASTER.foo.10.bar.baz") assert labels == { "tron.yelp.com/run_num": "10", } def test_build_labels_with_attempt_number_zero(self): labels = action_runner.build_labels("MASTER.foo.10.bar", attempt_number=0) assert labels == { "tron.yelp.com/run_num": "10", "tron.yelp.com/attempt_number": "0", } def test_build_labels_with_attempt_number_retry(self): labels = action_runner.build_labels("MASTER.foo.10.bar", attempt_number=2) assert labels == { "tron.yelp.com/run_num": "10", "tron.yelp.com/attempt_number": "2", } def test_build_labels_with_attempt_number_and_original_labels(self): current_labels = {"LABEL1": "value_1"} labels = action_runner.build_labels("MASTER.foo.10.bar", current_labels, attempt_number=1) assert labels == { "tron.yelp.com/run_num": "10", "tron.yelp.com/attempt_number": "1", "LABEL1": "value_1", } def test_build_labels_without_attempt_number_omits_label(self): labels = action_runner.build_labels("MASTER.foo.10.bar") assert "tron.yelp.com/attempt_number" not in labels ================================================ FILE: tests/bin/action_status_test.py ================================================ import signal import tempfile from unittest import mock from testifycompat import setup_teardown from testifycompat import TestCase from tron import yaml from tron.bin import action_status class TestActionStatus(TestCase): @setup_teardown def setup_status_file(self): self.status_file = tempfile.NamedTemporaryFile(mode="r+") self.status_content = { "pid": 1234, "return_code": None, "run_id": "MASTER.foo.bar.1234", } self.status_file.write(yaml.safe_dump(self.status_content)) self.status_file.flush() self.status_file.seek(0) yield self.status_file.close() @mock.patch("tron.bin.action_status.os.killpg", autospec=True) @mock.patch( "tron.bin.action_status.os.getpgid", autospec=True, return_value=42, ) def test_send_signal(self, mock_getpgid, mock_kill): action_status.send_signal(signal.SIGKILL, self.status_file) mock_getpgid.assert_called_with(self.status_content["pid"]) mock_kill.assert_called_with(42, signal.SIGKILL) def test_get_field_retrieves_last_entry(self): self.status_file.seek(0, 2) additional_status_content = { "pid": 1234, "return_code": 0, "run_id": "MASTER.foo.bar.1234", "command": "echo " + "really_long" * 100, } self.status_file.write( yaml.safe_dump(additional_status_content, explicit_start=True), ) self.status_file.flush() self.status_file.seek(0) assert action_status.get_field("return_code", self.status_file) == 0 def test_get_field_none(self): assert action_status.get_field("return_code", self.status_file) is None ================================================ FILE: tests/bin/check_tron_jobs_test.py ================================================ import time from unittest import mock from unittest.mock import patch from unittest.mock import PropertyMock import pytest from testifycompat import assert_equal from testifycompat import TestCase from tron.bin import check_tron_jobs from tron.bin.check_tron_jobs import State @pytest.fixture(autouse=True) def mock_run_interval(): with patch.object(check_tron_jobs, "_run_interval", 300): yield class TestCheckJobs(TestCase): @patch("tron.bin.check_tron_jobs.check_job_result", autospec=True) @patch("tron.bin.check_tron_jobs.Client", autospec=True) @patch("tron.bin.check_tron_jobs.cmd_utils", autospec=True) @patch("tron.bin.check_tron_jobs.parse_cli", autospec=True) def test_check_job_result_exception( self, mock_args, mock_cmd_utils, mock_client, mock_check_job_result, ): type(mock_args.return_value).job = PropertyMock(return_value=None) mock_client.return_value.jobs.return_value = [ { "name": "job1", }, { "name": "job2", }, { "name": "job3", }, ] mock_check_job_result.side_effect = [ KeyError("foo"), None, TypeError, ] error_code = check_tron_jobs.main() assert_equal(error_code, 1) assert_equal(mock_check_job_result.call_count, 3) # These tests test job run succeeded scenarios def test_job_succeeded(self): job_runs = { "status": "running", "next_run": None, "runs": [ { "id": "MASTER.test.3", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "end_time": None, }, { "id": "MASTER.test.2", "state": "running", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, }, { "id": "MASTER.test.1", "state": "succeeded", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1700), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.1") assert_equal(state, State.SUCCEEDED) def test_job_running_and_action_succeeded(self): job_runs = { "status": "running", "next_run": None, "runs": [ { "id": "MASTER.test.3", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "end_time": None, }, { "id": "MASTER.test.2", "state": "running", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, "runs": [ { "id": "MASTER.test.2.action2", "state": "running", }, { "id": "MASTER.test.1.action1", "state": "succeeded", }, ], # noqa: E122 }, { "id": "MASTER.test.1", "state": "succeeded", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1700), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.1") assert_equal(state, State.SUCCEEDED) def test_get_relevant_action_picks_the_first_one_succeeded(self): action_runs = [ { "id": "MASTER.test.action1", "action_name": "action1", "state": "succeeded", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1200), ), "duration": "0:18:01.475067", }, { "id": "MASTER.test.action2", "action_name": "action2", "state": "succeeded", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "duration": "0:08:02.005783", }, { "id": "MASTER.test.action1", "action_name": "action1", "state": "succeeded", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time()), ), "duration": "0:00:01.006305", }, ] actual = check_tron_jobs.get_relevant_action( action_runs=action_runs, last_state=State.SUCCEEDED, actions_expected_runtime={ "action1": 86400.0, "action2": 86400.0, "action3": 86400.0, }, ) assert_equal(actual["id"], "MASTER.test.action1") # These tests test job run failed scenarios def test_job_failed(self): job_runs = { "status": "running", "next_run": None, "runs": [ { "id": "MASTER.test.3", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "end_time": None, }, { "id": "MASTER.test.2", "state": "running", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, }, { "id": "MASTER.test.1", "state": "failed", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1700), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.1") assert_equal(state, State.FAILED) def test_most_recent_end_time_job_failed(self): job_runs = { "status": "scheduled", "next_run": None, "runs": [ { "id": "MASTER.test.3", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "end_time": None, }, { "id": "MASTER.test.2", "state": "succeeded", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1700), ), }, { "id": "MASTER.test.1", "state": "failed", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 500), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.1") assert_equal(state, State.FAILED) def test_rerun_job_failed(self): job_runs = { "status": "scheduled", "next_run": None, "runs": [ { "id": "MASTER.test.4", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "end_time": None, }, { "id": "MASTER.test.3", "state": "failed", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 100), ), }, { "id": "MASTER.test.2", "state": "succeeded", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 500), ), }, { "id": "MASTER.test.1", "state": "failed", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1700), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.3") assert_equal(state, State.FAILED) def test_job_running_but_action_failed_already(self): job_runs = { "status": "running", "next_run": None, "runs": [ { "id": "MASTER.test.3", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "end_time": None, }, { "id": "MASTER.test.2", "state": "running", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, "runs": [ { "id": "MASTER.test.2.action2", "state": "running", }, { "id": "MASTER.test.1.action1", "state": "failed", }, ], # noqa: E122 }, { "id": "MASTER.test.1", "state": "succeeded", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1700), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.2") assert_equal(state, State.FAILED) def test_get_relevant_action_picks_the_one_that_failed(self): action_runs = [ { "node": { "username": "batch", "hostname": "localhost", "name": "localhost", "port": 22, }, "raw_command": "/bin/false", "requirements": [], "run_num": "582", "exit_status": 1, "stdout": None, "start_time": "2018-02-05 17:40:00", "id": "MASTER.kwatest.582.action1", "action_name": "action1", "state": "failed", "command": "/bin/false", "end_time": "2018-02-05 17:40:00", "stderr": None, "duration": "0:00:00.065018", "job_name": "MASTER.kwatest", }, { "node": { "username": "batch", "hostname": "localhost", "name": "localhost", "port": 22, }, "raw_command": "/bin/true", "requirements": [], "run_num": "582", "exit_status": 0, "stdout": None, "start_time": "2018-02-05 17:40:00", "id": "MASTER.kwatest.582.action2", "action_name": "action2", "state": "succeeded", "command": "/bin/true", "end_time": "2018-02-05 17:40:00", "stderr": None, "duration": "0:00:00.046243", "job_name": "MASTER.kwatest", }, ] actual = check_tron_jobs.get_relevant_action( action_runs=action_runs, last_state=State.FAILED, actions_expected_runtime={}, ) assert_equal(actual["state"], "failed") # These tests test job/action stuck scenarios def test_job_next_run_starting_no_overlap_is_stuck(self): job_runs = { "status": "running", "next_run": None, "runs": [ { "id": "MASTER.test.2", "state": "queued", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, }, { "id": "MASTER.test.1", "state": "running", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1200), ), "end_time": None, }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.1") assert_equal(state, State.STUCK) def test_job_next_run_starting_overlap_allowed_not_stuck(self): job_runs = { "status": "running", "next_run": None, "allow_overlap": True, "runs": [ { "id": "MASTER.test.3", "state": "queued", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, }, { "id": "MASTER.test.2", "state": "running", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1200), ), "end_time": None, }, { "id": "MASTER.test.1", "state": "succeeded", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1700), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.1") assert_equal(state, State.SUCCEEDED) def test_job_next_run_running_no_queueing_not_stuck(self): job_runs = { "status": "running", "next_run": None, "allow_overlap": False, "queueing": False, "runs": [ { "id": "MASTER.test.3", "state": "cancelled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, }, { "id": "MASTER.test.2", "state": "running", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1200), ), "end_time": None, }, { "id": "MASTER.test.1", "state": "succeeded", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1700), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.1") assert_equal(state, State.SUCCEEDED) def test_job_next_run_starting_no_queueing_not_stuck(self): job_runs = { "status": "starting", "next_run": None, "allow_overlap": False, "queueing": False, "runs": [ { "id": "MASTER.test.3", "state": "cancelled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, }, { "id": "MASTER.test.2", "state": "starting", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1200), ), "end_time": None, }, { "id": "MASTER.test.1", "state": "succeeded", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1700), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.1") assert_equal(state, State.SUCCEEDED) def test_job_running_job_exceeds_expected_runtime(self): job_runs = { "status": "running", "next_run": None, "expected_runtime": 480.0, "allow_overlap": True, "runs": [ { "id": "MASTER.test.100", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "end_time": None, "start_time": None, "duration": "", }, { "id": "MASTER.test.99", "state": "running", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, "duration": "0:10:01.883601", }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.99") assert_equal(state, State.STUCK) def test_job_starting_job_exceeds_expected_runtime(self): job_runs = { "status": "running", "next_run": None, "expected_runtime": 480.0, "allow_overlap": True, "runs": [ { "id": "MASTER.test.100", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "end_time": None, "start_time": None, "duration": "", }, { "id": "MASTER.test.99", "state": "starting", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, "duration": "0:10:01.883601", }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.99") assert_equal(state, State.STUCK) def test_job_waiting_job_exceeds_expected_runtime_already_started(self): job_runs = { "status": "running", "next_run": None, "expected_runtime": 480.0, "allow_overlap": True, "runs": [ { "id": "MASTER.test.100", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "end_time": None, "start_time": None, "duration": "", }, { "id": "MASTER.test.99", "state": "waiting", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, "duration": "0:10:01.883601", }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.99") assert_equal(state, State.STUCK) def test_job_running_action_exceeds_expected_runtime(self): job_runs = { "status": "running", "next_run": None, "actions_expected_runtime": { "action1": 720.0, "action2": 480.0, }, "runs": [ dict( id="MASTER.test.3", state="scheduled", run_time=time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), end_time=None, duration="", ), dict( id="MASTER.test.2", state="running", run_time=time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), end_time=None, duration="0:10:01.883601", runs=[ dict( id="MASTER.test.2.action2", state="running", action_name="action2", start_time=time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), duration="0:10:01.883601", ), dict( id="MASTER.test.2.action1", state="running", action_name="action1", start_time=time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), duration="0:10:01.885401", ), ], ), dict( id="MASTER.test.1", state="succeeded", run_time=time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), end_time=time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1700), ), duration="0:15:00.453601", ), ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.2") assert_equal(state, State.STUCK) def test_job_running_action_exceeds_expected_runtime_and_other_action_failed(self): job_runs = { "status": "running", "next_run": None, "actions_expected_runtime": { "action1": 720.0, "action2": 480.0, }, "runs": [ dict( id="MASTER.test.1", state="running", run_time=time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), end_time=None, duration="0:10:01.883601", runs=[ dict( id="MASTER.test.1.action2", state="failed", action_name="action2", start_time=time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), duration="0:10:01.883601", ), dict( id="MASTER.test.1.action1", state="running", action_name="action1", start_time=time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), duration="0:10:01.885401", ), ], ), ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.1") assert_equal(state, State.FAILED) def test_job_stuck_when_runtime_not_sorted(self): job_runs = { "status": "running", "next_run": None, "runs": [ { "id": "MASTER.test.2", "state": "running", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, }, { "id": "MASTER.test.1", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time()), ), "end_time": None, }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.2") assert_equal(state, State.STUCK) def test_get_relevant_action_pick_the_one_stuck(self): action_runs = [ { "id": "MASTER.test.1.action3", "state": "succeeded", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1200), ), "duration": "0:18:01.475067", }, { "id": "MASTER.test.1.action2", "state": "running", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1100), ), "duration": "0:18:40.005783", }, { "id": "MASTER.test.1.action1", "state": "succeeded", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1000), ), "duration": "0:00:01.006305", }, ] actual = check_tron_jobs.get_relevant_action( action_runs=action_runs, last_state=State.STUCK, actions_expected_runtime={ "action1": 86400.0, "action2": 86400.0, "action3": 86400.0, }, ) assert_equal(actual["id"], "MASTER.test.1.action2") def test_get_relevant_action_pick_the_one_exceeds_expected_runtime(self): action_runs = [ { "id": "MASTER.test.1.action3", "state": "running", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "duration": "0:10:00.006305", }, { "id": "MASTER.test.1.action2", "state": "running", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "duration": "0:10:00.006383", }, { "id": "MASTER.test.1.action1", "state": "succeeded", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "duration": "0:10:00.006331", }, ] actions_expected_runtime = { "action3": 480.0, "action2": 720.0, "action1": 900.0, } actual = check_tron_jobs.get_relevant_action( action_runs=action_runs, last_state=State.STUCK, actions_expected_runtime=actions_expected_runtime, ) assert_equal(actual["id"], "MASTER.test.1.action3") def test_get_relevant_action_pick_the_one_starting(self): action_runs = [ { "id": "MASTER.test.1.action3", "state": "starting", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "duration": "0:10:00.006305", }, { "id": "MASTER.test.1.action2", "state": "running", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "duration": "0:10:00.006383", }, { "id": "MASTER.test.1.action1", "state": "succeeded", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "duration": "0:10:00.006331", }, ] actions_expected_runtime = { "action3": 480.0, "action2": 720.0, "action1": 900.0, } actual = check_tron_jobs.get_relevant_action( action_runs=action_runs, last_state=State.STUCK, actions_expected_runtime=actions_expected_runtime, ) assert_equal(actual["id"], "MASTER.test.1.action3") def test_get_relevant_action_pick_the_one_exceeds_expected_runtime_with_long_duration( self, ): action_runs = [ { "id": "MASTER.test.1.action3", "action_name": "action3", "state": "running", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "duration": "1 day, 0:10:00.006305", }, { "id": "MASTER.test.1.action2", "action_name": "action2", "state": "running", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "duration": "2 days, 0:10:00.006383", }, { "id": "MASTER.test.1.action1", "action_name": "action1", "state": "running", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "duration": "1 day, 0:10:00.006331", }, ] actions_expected_runtime = { "action3": 100000.0, "action2": 100000.0, "action1": 100000.0, } actual = check_tron_jobs.get_relevant_action( action_runs=action_runs, last_state=State.STUCK, actions_expected_runtime=actions_expected_runtime, ) assert_equal(actual["id"], "MASTER.test.1.action2") def test_no_job_scheduled_or_queuing(self): """If the past 2 runs succeeded but no future job is scheuled, we should consider the job to have suceeded. """ job_runs = { "status": "succeeded", "next_run": None, "runs": [ { "id": "MASTER.test.2", "state": "succeeded", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 300), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), }, { "id": "MASTER.test.1", "state": "succeeded", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 900), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1200), ), }, ], "monitoring": {}, } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.2") assert_equal(state, State.SUCCEEDED) # These tests test job without succeeded/failed run scenarios def test_job_no_runs_to_check(self): job_runs = { "status": "scheduled", "next_run": None, "runs": [ { "id": "MASTER.test.1", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 1200), ), "end_time": None, }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.1") assert_equal(state, State.NO_RUNS_TO_CHECK) def test_job_has_no_runs_at_all(self): job_runs = { "status": "running", "next_run": None, "runs": [], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run, None) assert_equal(state, State.NO_RUN_YET) # These tests test job/action unknown scenarios def test_job_unknown(self): job_runs = { "status": "running", "next_run": None, "runs": [ { "id": "MASTER.test.3", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, }, { "id": "MASTER.test.2", "state": "unknown", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1200), ), "end_time": None, }, { "id": "MASTER.test.1", "state": "succeeded", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1700), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.2") assert_equal(state, State.UNKNOWN) def test_job_running_but_action_unknown_already(self): job_runs = { "status": "running", "next_run": None, "runs": [ { "id": "MASTER.test.3", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "end_time": None, }, { "id": "MASTER.test.2", "state": "running", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, "runs": [ { "id": "MASTER.test.2.action2", "state": "running", }, { "id": "MASTER.test.1.action1", "state": "unknown", }, ], # noqa: E122 }, { "id": "MASTER.test.1", "state": "succeeded", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1700), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.2") assert_equal(state, State.UNKNOWN) def test_job_waiting_but_action_unknown_already(self): job_runs = { "status": "waiting", "next_run": None, "runs": [ { "id": "MASTER.test.3", "state": "scheduled", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "end_time": None, }, { "id": "MASTER.test.2", "state": "waiting", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "end_time": None, "runs": [ { "id": "MASTER.test.2.action2", "state": "waiting", }, { "id": "MASTER.test.1.action1", "state": "unknown", }, ], # noqa: E122 }, { "id": "MASTER.test.1", "state": "succeeded", "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), "end_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1700), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run["id"], "MASTER.test.2") assert_equal(state, State.UNKNOWN) # These tests test guess realert feature def test_guess_realert_every(self): job_runs = { "status": "running", "next_run": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "runs": [ { "id": "MASTER.test.3", "state": "scheduled", "start_time": None, "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), }, { "id": "MASTER.test.2", "state": "failed", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), }, { "id": "MASTER.test.1", "state": "succeeded", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), }, ], # noqa: E122 } realert_every = check_tron_jobs.guess_realert_every(job_runs) assert_equal(realert_every, 4) def test_guess_realert_every_no_action_run_starts(self): job_runs = { "status": "running", "next_run": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "runs": [ { "id": "MASTER.test.3", "state": "scheduled", "start_time": None, }, { "id": "MASTER.test.2", "state": "failed", "start_time": None, "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 200), ), }, { "id": "MASTER.test.1", "state": "succeeded", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), "run_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), }, ], # noqa: E122 } realert_every = check_tron_jobs.guess_realert_every(job_runs) assert_equal(realert_every, 2) def test_guess_realert_every_queue_job(self): job_runs = { "status": "running", "next_run": None, "runs": [ { "id": "MASTER.test.3", "state": "queued", "start_time": None, }, { "id": "MASTER.test.2", "state": "running", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 600), ), }, { "id": "MASTER.test.1", "state": "succeeded", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 1800), ), }, ], } realert_every = check_tron_jobs.guess_realert_every(job_runs) assert_equal(realert_every, -1) def test_guess_realert_every_frequent_run(self): job_runs = { "status": "running", "next_run": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 10), ), "runs": [ { "id": "MASTER.test.3", "state": "scheduled", "start_time": None, }, { "id": "MASTER.test.2", "state": "failed", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 10), ), }, { "id": "MASTER.test.1", "state": "succeeded", "start_time": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 20), ), }, ], # noqa: E122 } realert_every = check_tron_jobs.guess_realert_every(job_runs) assert_equal(realert_every, 1) def test_guess_realert_every_first_time_job(self): job_runs = { "status": "enabled", "next_run": time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + 600), ), "runs": [ { "id": "MASTER.test.1", "state": "scheduled", "start_time": None, }, ], } realert_every = check_tron_jobs.guess_realert_every(job_runs) assert_equal(realert_every, -1) class TestCheckPreciousJobs: @pytest.fixture(autouse=True) def setup_job(self): self.job_name = "fake_job" self.monitoring = { "team": "fake_team", "notification_email": "fake_email", check_tron_jobs.PRECIOUS_JOB_ATTR: True, } self.runs = [ { "id": f"{self.job_name}.15", "job_name": self.job_name, "run_num": 15, "run_time": "2018-10-13 12:00:00", "start_time": "2018-10-13 12:00:00", "end_time": "2018-10-13 12:30:00", "state": "succeeded", "exit_status": 0, }, { "id": f"{self.job_name}.14", "job_name": self.job_name, "run_num": 14, "run_time": "2018-10-12 12:00:00", "start_time": "2018-10-12 12:00:00", "end_time": "2018-10-12 12:30:00", "state": "failed", "exit_status": 1, }, { "id": f"{self.job_name}.13", "job_name": self.job_name, "run_num": 13, "run_time": "2018-10-11 13:00:00", "start_time": "2018-10-11 13:00:00", "end_time": "2018-10-11 13:30:00", "state": "failed", "exit_status": 1, }, { "id": f"{self.job_name}.12", "job_name": self.job_name, "run_num": 12, "run_time": "2018-10-11 12:00:00", "start_time": "2018-10-11 12:00:00", "end_time": "2018-10-11 12:30:00", "state": "succeeded", "exit_status": 0, }, { "id": f"{self.job_name}.11", "job_name": self.job_name, "run_num": 11, "run_time": "2018-10-10 13:00:00", "start_time": "2018-10-10 13:00:00", "end_time": "2018-10-10 13:30:00", "state": "succeeded", "exit_status": 0, }, { "id": f"{self.job_name}.10", "job_name": self.job_name, "run_num": 10, "run_time": "2018-10-10 12:00:00", "start_time": "2018-10-10 12:00:00", "end_time": "2018-10-10 12:30:00", "state": "failed", "exit_status": 1, }, ] self.job = { "name": "fake_job", "status": "enabled", "monitoring": self.monitoring, "runs": self.runs, } @patch("time.time", mock.Mock(return_value=1539460800.0), autospec=None) def test_sort_runs_by_interval_day(self): run_buckets = check_tron_jobs.sort_runs_by_interval(self.job, "day") assert set(run_buckets.keys()) == { "2018.10.10", "2018.10.11", "2018.10.12", "2018.10.13", } assert len(run_buckets["2018.10.10"]) == 2 assert len(run_buckets["2018.10.11"]) == 2 assert len(run_buckets["2018.10.12"]) == 1 assert len(run_buckets["2018.10.13"]) == 1 @patch("time.time", mock.Mock(return_value=1539633600.0), autospec=None) def test_sort_runs_by_interval_day_empty_buckets(self): self.job["runs"] = [ { "id": f"{self.job_name}.16", "job_name": self.job_name, "run_num": 16, "run_time": "2018-10-15 12:00:00", "start_time": "2018-10-15 12:00:00", "end_time": "2018-10-15 12:30:00", "state": "succeeded", "exit_status": 0, } ] + self.job["runs"] run_buckets = check_tron_jobs.sort_runs_by_interval(self.job, "day") assert "2018.10.14" in run_buckets assert run_buckets["2018.10.14"] == [] @patch("time.time", mock.Mock(return_value=1539633600.0), autospec=None) def test_sort_runs_by_interval_day_old_empty_buckets(self): # If the newest run is a backfill for an older date, shouldn't be # included for buckets self.job["runs"] = [ { "id": f"{self.job_name}.16", "job_name": self.job_name, "run_num": 16, "run_time": "2018-10-01 12:00:00", "start_time": "2018-10-01 12:00:00", "end_time": "2018-10-01 12:30:00", "state": "succeeded", "exit_status": 0, } ] + self.job["runs"] run_buckets = check_tron_jobs.sort_runs_by_interval(self.job, "day") # Current time is patched to 2018-10-15, so we should include # NUM_PRECIOUS intervals, starting 2018-10-09 # 2018-10-01 should not be included, even though there is a run for it, # because it is too old assert "2018.10.01" not in run_buckets assert run_buckets["2018.10.14"] == [] assert len(run_buckets) == 7 @patch( "tron.bin.check_tron_jobs.guess_realert_every", mock.Mock(return_value=1), autospec=None, ) @patch("tron.bin.check_tron_jobs.Client", autospec=True) @patch("tron.bin.check_tron_jobs.compute_check_result_for_job_runs", autospec=True) @patch("tron.bin.check_tron_jobs.get_object_type_from_identifier", autospec=True) def test_compute_check_result_for_job_not_precious( self, mock_get_obj_type, mock_check_job_runs, mock_client, ): client = mock_client("fake_server") client.cluster_name = "fake_cluster" del self.job["monitoring"][check_tron_jobs.PRECIOUS_JOB_ATTR] client.job = mock.Mock(return_value=self.job) mock_check_job_runs.return_value = { "output": "fake_output", "status": "fake_status", } results = check_tron_jobs.compute_check_result_for_job( client, self.job, url_index={}, ) # make sure all job runs for a job are included by not incl count arg assert client.job.call_args_list == [ mock.call( mock_get_obj_type.return_value.url, include_action_runs=True, ), ] assert len(results) == 1 assert results[0]["name"] == "check_tron_job.fake_job" assert mock_check_job_runs.call_count == 1 @patch( "tron.bin.check_tron_jobs.guess_realert_every", mock.Mock(return_value=1), autospec=None, ) @patch("tron.bin.check_tron_jobs.Client", autospec=True) def test_compute_check_result_for_job_disabled(self, mock_client): client = mock_client("fake_server") client.cluster_name = "fake_cluster" self.job["status"] = "disabled" results = check_tron_jobs.compute_check_result_for_job( client, self.job, url_index={}, ) assert len(results) == 1 assert results[0]["status"] == 0 assert results[0]["output"] == "OK: fake_job is disabled and won't be checked." @patch( "tron.bin.check_tron_jobs.guess_realert_every", mock.Mock(return_value=1), autospec=None, ) @patch("time.time", mock.Mock(return_value=1539460800.0), autospec=None) @patch("tron.bin.check_tron_jobs.Client", autospec=True) @patch("tron.bin.check_tron_jobs.compute_check_result_for_job_runs", autospec=True) @patch("tron.bin.check_tron_jobs.get_object_type_from_identifier", autospec=True) def test_compute_check_result_for_job_enabled( self, mock_get_obj_type, mock_check_job_runs, mock_client, ): client = mock_client("fake_server") client.cluster_name = "fake_cluster" self.job["monitoring"]["check_every"] = 500 self.job["monitoring"]["hide_stderr"] = True client.job = mock.Mock(return_value=self.job) mock_check_job_runs.return_value = { "output": "fake_output", "status": "fake_status", } results = check_tron_jobs.compute_check_result_for_job( client, self.job, url_index={}, ) # Test that hide_stderr is passed to check_job_runs assert mock_check_job_runs.call_args_list[0][1]["hide_stderr"] is True # make sure all job runs for a job are included by not incl count arg assert client.job.call_args_list == [ mock.call( mock_get_obj_type.return_value.url, include_action_runs=True, ), ] assert len(results) == 4 assert {res["name"] for res in results} == { "check_tron_job.fake_job-2018.10.10", "check_tron_job.fake_job-2018.10.11", "check_tron_job.fake_job-2018.10.12", "check_tron_job.fake_job-2018.10.13", } for res in results: assert res["check_every"] == "300s" assert check_tron_jobs.PRECIOUS_JOB_ATTR not in res ================================================ FILE: tests/bin/get_tron_metrics_test.py ================================================ import subprocess from unittest import mock import pytest from tron.bin import get_tron_metrics def test_send_data_metric(): process = mock.Mock() process.communicate = mock.Mock(return_value=(b"fake_output", b"fake_error")) cmd_str = "meteorite data -v fake_name fake_metric_type fake_value " "-d fake_dim_key:fake_dim_value" with mock.patch( "subprocess.Popen", mock.Mock(return_value=process), autospec=None, ) as mock_popen: get_tron_metrics.send_data_metric( name="fake_name", metric_type="fake_metric_type", value="fake_value", dimensions={"fake_dim_key": "fake_dim_value"}, dry_run=False, ) assert mock_popen.call_count == 1 assert mock_popen.call_args == mock.call( cmd_str.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) def test_send_data_metric_dry_run(): with mock.patch("subprocess.Popen", autospec=True) as mock_popen: get_tron_metrics.send_data_metric( name="fake_name", metric_type="fake_metric_type", value="fake_value", dimensions="fake_dimensions", dry_run=True, ) assert mock_popen.call_count == 0 @mock.patch("tron.bin.get_tron_metrics.send_data_metric", autospec=True) def test_send_counter(mock_send_data_metric): kwargs = dict(count="fake_count") get_tron_metrics.send_counter("fake_name", **kwargs) assert mock_send_data_metric.call_count == 1 assert mock_send_data_metric.call_args == mock.call( name="fake_name", metric_type="counter", value="fake_count", dimensions={}, dry_run=False, ) @mock.patch("tron.bin.get_tron_metrics.send_data_metric", autospec=True) def test_send_gauge(mock_send_data_metric): kwargs = dict(value="fake_value") get_tron_metrics.send_gauge("fake_name", **kwargs) assert mock_send_data_metric.call_count == 1 assert mock_send_data_metric.call_args == mock.call( name="fake_name", metric_type="gauge", value="fake_value", dimensions={}, dry_run=False, ) @mock.patch("tron.bin.get_tron_metrics.send_counter", autospec=True) def test_send_meter(mock_send_counter): get_tron_metrics.send_meter("fake_name") assert mock_send_counter.call_count == 1 assert mock_send_counter.call_args == mock.call("fake_name") @mock.patch("tron.bin.get_tron_metrics.send_gauge", autospec=True) def test_send_histogram(mock_send_gauge): kwargs = dict( p50="fake_p50", p75="fake_p75", p95="fake_p95", p99="fake_p99", ) p50_kwargs = dict( **kwargs, value="fake_p50", ) get_tron_metrics.send_histogram("fake_name", **kwargs) assert mock_send_gauge.call_count == len(kwargs) assert mock_send_gauge.call_args_list[0] == mock.call( "fake_name.p50", **p50_kwargs, ) @mock.patch("tron.bin.get_tron_metrics.send_meter", autospec=True) @mock.patch("tron.bin.get_tron_metrics.send_histogram", autospec=True) def test_send_timer(mock_send_meter, mock_send_histogram): get_tron_metrics.send_timer("fake_name") assert mock_send_meter.call_count == 1 assert mock_send_meter.call_args == mock.call("fake_name") assert mock_send_histogram.call_count == 1 assert mock_send_histogram.call_args == mock.call("fake_name") @pytest.mark.parametrize("cluster", ["fake_cluster", None]) def test_send_metrics(cluster): mock_send_counter = mock.Mock() metrics = dict(counter=[dict(name="fake_name")]) with mock.patch( "tron.bin.get_tron_metrics._METRIC_SENDERS", dict(counter=mock_send_counter), autospec=None, ): get_tron_metrics.send_metrics(metrics, cluster=cluster, dry_run=True) assert mock_send_counter.call_count == 1 if cluster: assert mock_send_counter.call_args == mock.call( "fake_name", dry_run=True, dimensions={"tron_cluster": "fake_cluster"}, ) else: assert mock_send_counter.call_args == mock.call("fake_name", dry_run=True) ================================================ FILE: tests/bin/recover_batch_test.py ================================================ import tempfile from queue import Queue from unittest import mock import pytest from tron.bin import recover_batch from tron.bin.action_runner import StatusFile @pytest.fixture def mock_file(): f = tempfile.NamedTemporaryFile() yield f.name f.close() @mock.patch.object(recover_batch, "reactor") @mock.patch("tron.bin.recover_batch.get_exit_code", autospec=True) @pytest.mark.parametrize( "exit_code,error_msg,should_stop", [ (1, "failed", True), (None, None, False), ], ) def test_notify(mock_get_exit_code, mock_reactor, exit_code, error_msg, should_stop): mock_get_exit_code.return_value = exit_code, error_msg queue = Queue() path = mock.Mock() recover_batch.notify(queue, "some_ignored", path, "mask") if should_stop: assert mock_reactor.stop.call_count == 1 assert queue.get_nowait() == (exit_code, error_msg) else: assert mock_reactor.stop.call_count == 0 assert queue.empty() @mock.patch("tron.bin.recover_batch.psutil.pid_exists", autospec=True) @mock.patch("tron.bin.recover_batch.read_last_yaml_entries", autospec=True) @pytest.mark.parametrize( "line,exit_code,is_running,error_msg", [ ( {"return_code": 0, "runner_pid": 12345}, 0, False, None, ), # action runner finishes successfully ( # action runner is killed {"return_code": -9, "runner_pid": 12345}, 9, False, "Action run killed by signal SIGKILL", ), ( # No return code but action_runner pid is not running {"runner_pid": 12345}, 1, False, "Action runner pid 12345 no longer running. Assuming an exit of 1.", ), ( {"runner_pid": 12345}, None, True, None, ), # No return code but action_runner pid is running ( {}, None, Exception, None, ), # No return code or PID from the file ], ) def test_get_exit_code( mock_read_last_yaml_entries, mock_pid_running, line, exit_code, is_running, error_msg, ): fake_path = "/file/path" mock_read_last_yaml_entries.return_value = line mock_pid_running.side_effect = [is_running] actual_exit_code, actual_error_msg = recover_batch.get_exit_code(fake_path) assert actual_exit_code == exit_code assert actual_error_msg == error_msg def test_read_last_yaml_roundtrip(mock_file): """Check that read_last_yaml_entries returns the same thing that the action runner wrote.""" status = StatusFile(mock_file) expected_content = [ {"return_code": None, "pid": 10345, "command": "foo"}, {"return_code": 1, "pid": 10345, "command": "foo"}, ] with mock.patch.object(status, "get_content", side_effect=expected_content): with status.wrap(command="echo hello", run_id="job.1.action", proc=mock.Mock()): # In the context manager, we've written the first value of get_content. first = recover_batch.read_last_yaml_entries(mock_file) assert first == expected_content[0] # After, we write another status entry. We should return the latest. second = recover_batch.read_last_yaml_entries(mock_file) assert second == expected_content[1] @mock.patch.object(recover_batch, "reactor") @mock.patch("tron.bin.recover_batch.Queue", autospec=True) @mock.patch("tron.bin.recover_batch.get_exit_code", autospec=True, return_value=(None, None)) @mock.patch("tron.bin.recover_batch.StatusFileWatcher", autospec=True) @pytest.mark.parametrize("existing_code,watcher_code", [(None, 1), (123, None)]) def test_run( mock_watcher, mock_get_exit_code, mock_queue, mock_reactor, existing_code, watcher_code, ): mock_get_exit_code.return_value = (existing_code, "") mock_queue.return_value.get.return_value = (watcher_code, "") mock_path = mock.Mock() if existing_code is not None: expected = existing_code else: expected = watcher_code with pytest.raises(SystemExit) as e: recover_batch.run(mock_path) assert e.code == expected assert mock_get_exit_code.call_args_list == [mock.call(mock_path)] if existing_code is not None: assert mock_watcher.call_count == 0 else: assert mock_watcher.call_count == 1 ================================================ FILE: tests/command_context_test.py ================================================ import datetime from unittest import mock from testifycompat import assert_equal from testifycompat import assert_raises from testifycompat import run from testifycompat import setup from testifycompat import TestCase from tron import command_context from tron import node from tron import scheduler from tron.core import actionrun from tron.core import job from tron.core import jobrun from tron.core.jobrun import JobRunCollection class TestEmptyContext(TestCase): @setup def build_context(self): self.context = command_context.CommandContext(None) def test__getitem__(self): assert_raises(KeyError, self.context.__getitem__, "foo") def test_get(self): assert not self.context.get("foo") class TestBuildFilledContext(TestCase): def test_build_filled_context_no_objects(self): output = command_context.build_filled_context() assert not output.base assert not output.next def test_build_filled_context_single(self): output = command_context.build_filled_context( command_context.JobContext, ) assert isinstance(output.base, command_context.JobContext) assert not output.next def test_build_filled_context_chain(self): objs = [command_context.JobContext, command_context.JobRunContext] output = command_context.build_filled_context(*objs) assert isinstance(output.base, objs[1]) assert isinstance(output.next.base, objs[0]) assert not output.next.next class SimpleContextTestCaseBase(TestCase): __test__ = False def test_hit(self): assert_equal(self.context["foo"], "bar") def test_miss(self): assert_raises(KeyError, self.context.__getitem__, "your_mom") def test_get_hit(self): assert_equal(self.context.get("foo"), "bar") def test_get_miss(self): assert not self.context.get("unknown") class SimpleDictContextTestCase(SimpleContextTestCaseBase): @setup def build_context(self): self.context = command_context.CommandContext(dict(foo="bar")) class SimpleObjectContextTestCase(SimpleContextTestCaseBase): @setup def build_context(self): class Obj: foo = "bar" self.context = command_context.CommandContext(Obj) class ChainedDictContextTestCase(SimpleContextTestCaseBase): @setup def build_context(self): self.next_context = command_context.CommandContext( dict(foo="bar", next_foo="next_bar"), ) self.context = command_context.CommandContext( dict(), self.next_context, ) def test_chain_get(self): assert_equal(self.context["next_foo"], "next_bar") class ChainedDictOverrideContextTestCase(SimpleContextTestCaseBase): @setup def build_context(self): self.next_context = command_context.CommandContext( dict(foo="your mom", next_foo="next_bar"), ) self.context = command_context.CommandContext( dict(foo="bar"), self.next_context, ) def test_chain_get(self): assert_equal(self.context["next_foo"], "next_bar") class ChainedObjectOverrideContextTestCase(SimpleContextTestCaseBase): @setup def build_context(self): class MyObject(TestCase): pass obj = MyObject() obj.foo = "bar" self.next_context = command_context.CommandContext( dict(foo="your mom", next_foo="next_bar"), ) self.context = command_context.CommandContext(obj, self.next_context) def test_chain_get(self): assert_equal(self.context["next_foo"], "next_bar") class TestJobContext(TestCase): @setup def setup_job(self): self.last_success = mock.Mock(run_time=datetime.datetime(2012, 3, 14)) mock_scheduler = mock.create_autospec(scheduler.GeneralScheduler) run_collection = mock.create_autospec( JobRunCollection, last_success=self.last_success, ) self.job = job.Job( "MASTER.jobname", mock_scheduler, run_collection=run_collection, ) self.context = command_context.JobContext(self.job) def test_name(self): assert_equal(self.context.name, self.job.name) def test__getitem__last_success(self): item = self.context["last_success#day-1"] expected_date = self.last_success.run_time - datetime.timedelta(days=1) assert_equal(item, str(expected_date.day)) item = self.context["last_success#shortdate"] assert_equal(item, "2012-03-14") def test__getitem__last_success_bad_date_spec(self): name = "last_success#beers-3" assert_raises(KeyError, lambda: self.context[name]) def test__getitem__last_success_bad_date_name(self): name = "first_success#shortdate-1" assert_raises(KeyError, lambda: self.context[name]) def test__getitem__last_success_no_date_spec(self): name = "last_success" assert_raises(KeyError, lambda: self.context[name]) def test__getitem__missing(self): assert_raises(KeyError, lambda: self.context["bogus"]) def test_namespace(self): assert self.context.namespace == "MASTER" class TestJobRunContext(TestCase): @setup def setup_context(self): self.jobrun = mock.create_autospec(jobrun.JobRun, run_time="sometime", manual=True) self.context = command_context.JobRunContext(self.jobrun) def test_cleanup_job_status(self): self.jobrun.action_runs.is_failed = False self.jobrun.action_runs.is_complete_without_cleanup = True assert_equal(self.context.cleanup_job_status, "SUCCESS") def test_cleanup_job_status_failure(self): self.jobrun.action_runs.is_failed = True assert_equal(self.context.cleanup_job_status, "FAILURE") def test_runid(self): assert_equal(self.context.runid, self.jobrun.id) def test_manual_run(self): assert self.context.manual == "true" @mock.patch("tron.command_context.timeutils.DateArithmetic", autospec=True) def test__getitem__(self, mock_date_math): name = "date_name" time_value = self.context[name] mock_date_math.parse.assert_called_with(name, self.jobrun.run_time) assert_equal(time_value, mock_date_math.parse.return_value) class TestActionRunContext(TestCase): @setup def build_context(self): mock_node = mock.create_autospec(node.Node, hostname="something") self.action_run = mock.create_autospec( actionrun.ActionRun, action_name="something", node=mock_node, ) self.context = command_context.ActionRunContext(self.action_run) def test_actionname(self): assert_equal(self.context.actionname, self.action_run.action_name) def test_node_hostname(self): assert_equal(self.context.node, self.action_run.node.hostname) class TestFiller(TestCase): @setup def setup_filler(self): self.filler = command_context.Filler() def test_filler_with_job__getitem__(self): context = command_context.JobContext(self.filler) todays_date = datetime.date.today().strftime("%Y-%m-%d") assert_equal(context["last_success#shortdate"], todays_date) def test_filler_with_job_run__getitem__(self): context = command_context.JobRunContext(self.filler) todays_date = datetime.date.today().strftime("%Y-%m-%d") assert_equal(context["shortdate"], todays_date) if __name__ == "__main__": run() ================================================ FILE: tests/commands/__init__.py ================================================ ================================================ FILE: tests/commands/backfill_test.py ================================================ import datetime from unittest import mock import pytest from tron.commands import backfill from tron.commands import client TEST_DATETIME_1 = datetime.datetime.strptime("2004-07-01", "%Y-%m-%d") TEST_DATETIME_2 = datetime.datetime.strptime("2004-07-02", "%Y-%m-%d") TEST_DATETIME_3 = datetime.datetime.strptime("2004-07-03", "%Y-%m-%d") @pytest.fixture(autouse=True) def mock_sleep(): async def empty_coro(*args, **kwargs): return None with mock.patch("asyncio.sleep", empty_coro, autospec=None): yield @pytest.fixture(autouse=True) def mock_client(): with mock.patch.object(client, "Client", autospec=True) as m: yield m @pytest.fixture(autouse=True) def mock_urlopen(): # prevent any requests from being made with mock.patch("urllib.request.urlopen", autospec=True) as m: yield m @pytest.fixture def mock_client_request(): with mock.patch.object(client, "request", autospec=True) as m: m.return_value = mock.Mock(error=False, content={}) # response yield m @pytest.fixture def fake_backfill_run(mock_client): tron_client = mock_client.return_value tron_client.url_base = "http://localhost" yield backfill.BackfillRun( tron_client, client.TronObjectIdentifier("JOB", "/a_job"), TEST_DATETIME_1, ) @pytest.mark.parametrize( "is_error,result,expected", [ (True, "an_error_msg", None), # tron api failed (False, "weird_resp_msg", None), # bad response, can't get job run name (False, "Created JobRun:real_job_run_name", "real_job_run_name"), # ok ], ) def test_backfill_run_create(mock_client_request, fake_backfill_run, event_loop, is_error, result, expected): mock_client_request.return_value.error = is_error mock_client_request.return_value.content["result"] = result assert expected == event_loop.run_until_complete(fake_backfill_run.create()) @pytest.mark.parametrize( "obj_type,expected", [ (client.RequestError(""), None), ([client.TronObjectIdentifier("JOB_RUN", "/a_run")], client.TronObjectIdentifier("JOB_RUN", "/a_run")), ], ) @mock.patch.object(client, "get_object_type_from_identifier", autospec=True) def test_backfill_run_get_run_id(mock_get_obj_type, fake_backfill_run, event_loop, obj_type, expected): mock_get_obj_type.side_effect = obj_type assert expected == event_loop.run_until_complete(fake_backfill_run.get_run_id()) assert expected == fake_backfill_run.run_id @pytest.mark.parametrize( "job_run_resp,expected", [ (client.RequestError, "unknown"), # polling failed ([{}], "unknown"), # default to unknown ([{"state": "failed"}], "failed"), # ok ], ) def test_backfill_run_sync_state(fake_backfill_run, event_loop, job_run_resp, expected): fake_backfill_run.run_id = client.TronObjectIdentifier("JOB_RUN", "/a_run") fake_backfill_run.tron_client.job_runs.side_effect = job_run_resp assert expected == event_loop.run_until_complete(fake_backfill_run.sync_state()) def test_backfill_run_watch_until_completion(fake_backfill_run, event_loop): async def change_run_state(): fake_backfill_run.run_state = "cancelled" fake_backfill_run.sync_state = change_run_state assert "cancelled" == event_loop.run_until_complete(fake_backfill_run.watch_until_completion()) @pytest.mark.parametrize( "run_id,response,expected", [ (None, mock.Mock(error=False), False), # no run_id (client.TronObjectIdentifier("JOB_RUN", "/a_run"), mock.Mock(error=True), False), # api error (client.TronObjectIdentifier("JOB_RUN", "/a_run"), mock.Mock(error=False), True), # ok ], ) def test_backfill_run_cancel( mock_client_request, fake_backfill_run, event_loop, run_id, response, expected, ): fake_backfill_run.run_id = run_id mock_client_request.return_value = response assert expected == event_loop.run_until_complete(fake_backfill_run.cancel()) @mock.patch("tron.commands.backfill.get_auth_token", lambda: "") @mock.patch.object(client, "get_object_type_from_identifier", autospec=True) def test_run_backfill_for_date_range_job_dne(mock_get_obj_type, event_loop): mock_get_obj_type.side_effect = ValueError with pytest.raises(ValueError): event_loop.run_until_complete( backfill.run_backfill_for_date_range("a_server", "a_job", []), ) @mock.patch("tron.commands.backfill.get_auth_token", lambda: "") @mock.patch.object(client, "get_object_type_from_identifier", autospec=True) def test_run_backfill_for_date_range_not_a_job(mock_get_obj_type, event_loop): mock_get_obj_type.return_value = client.TronObjectIdentifier("JOB_RUN", "a_url") with pytest.raises(ValueError): event_loop.run_until_complete( backfill.run_backfill_for_date_range("a_server", "a_job", []), ) @pytest.mark.parametrize( "ignore_errors,expected", [ (True, {"succeeded", "failed", "unknown"}), (False, {"succeeded", "failed", "not started"}), ], ) @mock.patch("tron.commands.backfill.get_auth_token", lambda: "") @mock.patch.object(client, "get_object_type_from_identifier", autospec=True) def test_run_backfill_for_date_range_normal(mock_get_obj_type, event_loop, ignore_errors, expected): run_states = (state for state in ["succeeded", "failed", "unknown"]) async def fake_run_until_completion(self): self.run_state = next(run_states) backfill.BackfillRun.run_until_completion = fake_run_until_completion dates = [TEST_DATETIME_1, TEST_DATETIME_2, TEST_DATETIME_3] mock_get_obj_type.return_value = client.TronObjectIdentifier("JOB", "a_url") backfill_runs = event_loop.run_until_complete( backfill.run_backfill_for_date_range( "a_server", "a_job", dates, max_parallel=2, ignore_errors=ignore_errors, ) ) assert {br.run_state for br in backfill_runs} == expected ================================================ FILE: tests/commands/client_test.py ================================================ from unittest import mock from urllib.error import HTTPError from urllib.error import URLError from testifycompat import assert_equal from testifycompat import assert_in from testifycompat import run from testifycompat import setup from testifycompat import setup_teardown from testifycompat import TestCase from tests.assertions import assert_raises from tests.testingutils import autospec_method from tron.commands import client from tron.commands.client import get_object_type_from_identifier from tron.commands.client import Response from tron.commands.client import TronObjectType def build_file_mock(content): return mock.Mock( read=mock.Mock(return_value=content), headers=mock.Mock(get_content_charset=mock.Mock(return_value="utf-8")), ) class TestRequest(TestCase): @setup def setup_options(self): self.url = "http://localhost:8089/jobs/" @setup_teardown def patch_urllib(self): patcher = mock.patch( "tron.commands.client.urllib.request.urlopen", autospec=True, ) with patcher as self.mock_urlopen: yield def test_build_url_request_no_data(self): request = client.build_url_request(self.url, None) assert request.has_header("User-agent") assert_equal(request.get_method(), "GET") assert_equal(request.get_full_url(), self.url) def test_build_url_request_with_data(self): data = {"param": "is_set", "other": 1} request = client.build_url_request(self.url, data) assert request.has_header("User-agent") assert_equal(request.get_method(), "POST") assert_equal(request.get_full_url(), self.url) assert_in("param=is_set", request.data.decode()) assert_in("other=1", request.data.decode()) @mock.patch("tron.commands.client.log", autospec=True) def test_load_response_content_success(self, _): content = b"not:valid:json" http_response = build_file_mock(content) response = client.load_response_content(http_response) assert_equal(response.error, client.DECODE_ERROR) assert_equal(response.content, content.decode("utf-8")) @mock.patch("tron.commands.client.log", autospec=True) def test_request_http_error(self, _): self.mock_urlopen.side_effect = HTTPError( self.url, 500, "broke", mock.Mock(get_content_charset=mock.Mock(return_value="utf-8")), build_file_mock(b"oops"), ) response = client.request(self.url) expected = client.Response(500, "broke", "oops") assert_equal(response, expected) @mock.patch("tron.commands.client.log", autospec=True) def test_request_url_error(self, _): self.mock_urlopen.side_effect = URLError("broke") response = client.request(self.url) expected = client.Response(client.URL_ERROR, "broke", None) assert_equal(response, expected) def test_request_success(self): self.mock_urlopen.return_value = build_file_mock(b'{"ok": "ok"}') response = client.request(self.url) expected = client.Response(None, None, {"ok": "ok"}) assert_equal(response, expected) class TestClientRequest(TestCase): @setup def setup_client(self): self.url = "http://localhost:8089/" self.client = client.Client(self.url) @setup_teardown def patch_request(self): with mock.patch( "tron.commands.client.request", autospec=True, ) as self.mock_request: yield def test_request_error(self): error_response = Response( error="404", msg="Not Found", content="big kahuna error", ) client.request = mock.Mock(return_value=error_response) exception = assert_raises( client.RequestError, self.client.request, "/jobs", ) assert str(exception) == error_response.content def test_request_success(self): ok_response = {"ok": "ok"} client.request.return_value = client.Response(None, None, ok_response) response = self.client.request("/jobs") assert_equal(response, ok_response) class TestClient(TestCase): @setup def setup_client(self): self.url = "http://localhost:8089/" self.client = client.Client(self.url) autospec_method(self.client.request) def test_config_post(self): name, data, hash = "name", "stuff", "hash" self.client.config(name, config_data=data, config_hash=hash) expected_data = { "config": data, "name": name, "hash": hash, "check": 0, } self.client.request.assert_called_with("/api/config", expected_data) def test_config_get_default(self): self.client.config("config_name") self.client.request.assert_called_with( "/api/config?name=config_name", ) def test_http_get(self): self.client.http_get("/api/jobs", {"include": 1}) self.client.request.assert_called_with("/api/jobs?include=1") def test_action_runs(self): self.client.action_runs("/api/jobs/name/0/act", num_lines=40) self.client.request.assert_called_with( "/api/jobs/name/0/act?include_stderr=1&include_stdout=1&num_lines=40", ) def test_job_runs(self): self.client.job_runs("/api/jobs/name/0") self.client.request.assert_called_with( "/api/jobs/name/0?include_action_graph=0&include_action_runs=1", ) def test_job(self): self.client.job("/api/jobs/name", count=20) self.client.request.assert_called_with( "/api/jobs/name?include_action_runs=0&num_runs=20", ) def test_jobs(self): self.client.jobs() self.client.request.assert_called_with( "/api/jobs?include_action_graph=1&include_action_runs=0&include_job_runs=0&include_node_pool=1", ) class TestUserAttribution(TestCase): def test_default_user_agent(self): url = "http://localhost:8089/" with mock.patch( "tron.commands.client.os.environ", autospec=True, ) as mock_environ: mock_environ.get.return_value = "testuser" default_client = client.Client(url, user_attribution=False) # we do not add user attribution by default assert "(testuser)" not in default_client.headers["User-Agent"] def test_attributed_user_agent(self): url = "http://localhost:8089/" with mock.patch( "tron.commands.client.os.environ", autospec=True, ) as mock_environ: mock_environ.get.return_value = "testuser" default_client = client.Client(url, user_attribution=True) # we do not add user attribution by default assert "(testuser)" in default_client.headers["User-Agent"] class TestGetUrl(TestCase): def test_get_job_url_for_action_run(self): url = client.get_job_url("MASTER.name.1.act") assert_equal(url, "/api/jobs/MASTER.name/1/act") def test_get_job_url_for_job(self): url = client.get_job_url("MASTER.name") assert_equal(url, "/api/jobs/MASTER.name") class TestGetContentFromIdentifier(TestCase): @setup def setup_client(self): self.options = mock.Mock() self.index = { "namespaces": ["OTHER", "MASTER"], "jobs": { "MASTER.namea": "", "MASTER.nameb": "", "OTHER.nameg": "", }, } def test_get_url_from_identifier_job_no_namespace(self): identifier = get_object_type_from_identifier(self.index, "namea") assert_equal(identifier.url, "/api/jobs/MASTER.namea") assert_equal(identifier.type, TronObjectType.job) def test_get_url_from_identifier_job(self): identifier = get_object_type_from_identifier( self.index, "MASTER.namea", ) assert_equal(identifier.url, "/api/jobs/MASTER.namea") assert_equal(identifier.type, TronObjectType.job) def test_get_url_from_identifier_job_run(self): identifier = get_object_type_from_identifier( self.index, "MASTER.nameb.7", ) assert_equal(identifier.url, "/api/jobs/MASTER.nameb/7") assert_equal(identifier.type, TronObjectType.job_run) def test_get_url_from_identifier_action_run(self): identifier = get_object_type_from_identifier( self.index, "MASTER.nameb.7.run", ) assert_equal(identifier.url, "/api/jobs/MASTER.nameb/7/run") assert_equal(identifier.type, TronObjectType.action_run) def test_get_url_from_identifier_job_no_namespace_not_master(self): identifier = get_object_type_from_identifier(self.index, "nameg") assert_equal(identifier.url, "/api/jobs/OTHER.nameg") assert_equal(identifier.type, TronObjectType.job) def test_get_url_from_identifier_no_match(self): exc = assert_raises( ValueError, get_object_type_from_identifier, self.index, "MASTER.namec", ) assert_in("namec", str(exc)) if __name__ == "__main__": run() ================================================ FILE: tests/commands/cmd_utils_test.py ================================================ import argparse from unittest import mock from testifycompat import assert_equal from testifycompat import assert_in from testifycompat import setup_teardown from testifycompat import TestCase from tron.commands import cmd_utils class TestGetConfig(TestCase): @setup_teardown def patch_environment(self): with mock.patch("tron.commands.cmd_utils.opener", autospec=True) as self.mock_opener, mock.patch( "tron.commands.cmd_utils.yaml", autospec=True ) as self.mock_yaml: yield def test_read_config_missing(self): self.mock_opener.side_effect = IOError assert_equal(cmd_utils.read_config(), {}) def test_read_config(self): assert_equal(cmd_utils.read_config(), self.mock_yaml.load.return_value) @mock.patch("tron.commands.cmd_utils.os.access", autospec=True) def test_get_client_config(self, mock_access): mock_access.return_value = False config = cmd_utils.get_client_config() assert_equal(mock_access.call_count, 2) assert_equal(config, {}) def test_filter_jobs_actions_runs_with_nothing(self): inputs = [ "M.foo", "M.foo.1", "M.foo.1.action1", "M.foo.2.action1", "M.bar", "M.bar.1.action", ] prefix = "" expected = ["M.foo", "M.bar"] assert_equal( cmd_utils.filter_jobs_actions_runs( prefix, inputs, ), expected, ) def test_filter_jobs_actions_runs_with_almost_a_job(self): inputs = [ "M.foo", "M.foo.1", "M.foo.1.action1", "M.foo.2.action1", "M.bar.1.action", ] prefix = "M.f" expected = ["M.foo"] assert_equal( cmd_utils.filter_jobs_actions_runs( prefix, inputs, ), expected, ) def test_filter_jobs_actions_runs_with_a_job_run(self): inputs = [ "M.foo", "M.foo.1", "M.foo.1.action1", "M.foo.2", "M.foo.2.action1", "M.bar.1.action", ] prefix = "M.foo." expected = ["M.foo.1", "M.foo.2"] assert_equal( cmd_utils.filter_jobs_actions_runs( prefix, inputs, ), expected, ) def test_filter_jobs_actions_runs_with_a_job_run_and_id(self): inputs = [ "M.foo", "M.foo.1", "M.foo.1.action1", "M.foo.2.action1", "M.bar.1.action", ] prefix = "M.foo.1" expected = ["M.foo.1", "M.foo.1.action1"] assert_equal( cmd_utils.filter_jobs_actions_runs( prefix, inputs, ), expected, ) class TestBuildOptionParser(TestCase): def test_build_option_parser(self): """Assert that we don't set default options so that we can load the defaults from the config. """ usage = "Something" epilog = "Something" argparse.ArgumentParser = mock.Mock() parser = cmd_utils.build_option_parser( usage=usage, epilog=epilog, ) argparse.ArgumentParser.assert_called_with( usage=usage, formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog, ) assert parser.add_argument.call_count == 5 args = [call[1] for call in parser.add_argument.mock_calls] expected = [ ("--version",), ("-v", "--verbose"), ("--server",), ("--cluster_name",), ("-s", "--save"), ] assert args == expected defaults = [call[2].get("default") for call in parser.add_argument.mock_calls] assert defaults == [None, None, None, None, None] class TestSuggestions(TestCase): def test_suggest_possibilities_none(self): expected = "" actual = cmd_utils.suggest_possibilities(word="FOO", possibilities=[]) assert_equal(actual, expected) def test_suggest_possibilities_many(self): expected = "FOOO, FOOBAR" actual = cmd_utils.suggest_possibilities( word="FOO", possibilities=["FOOO", "FOOBAR"], ) assert_in(expected, actual) def test_suggest_possibilities_one(self): expected = "FOOBAR?" actual = cmd_utils.suggest_possibilities( word="FOO", possibilities=["FOOBAR", "BAZ"], ) assert_in(expected, actual) ================================================ FILE: tests/commands/display_test.py ================================================ from unittest import mock from testifycompat import setup from testifycompat import setup_teardown from testifycompat import TestCase from tron.commands import display from tron.commands.display import DisplayActionRuns from tron.commands.display import DisplayJobRuns from tron.commands.display import DisplayJobs from tron.core import actionrun from tron.core import job class TestDisplayJobRuns(TestCase): @setup def setup_data(self): self.data = [ dict( id="something.23", state="FAIL", node=mock.MagicMock(), run_num=23, run_time="2012-01-20 23:11:23", start_time="2012-01-20 23:11:23", end_time="2012-02-21 23:10:10", duration="2 days", manual=False, ), dict( id="something.55", state="QUE", node=mock.MagicMock(), run_num=55, run_time="2012-01-20 23:11:23", start_time="2012-01-20 23:11:23", end_time="", duration="", manual=False, ), ] def test_format(self): out = DisplayJobRuns().format(self.data) lines = out.split("\n") assert len(lines) == 7 class TestDisplayJobs(TestCase): @setup def setup_data(self): self.data = [ dict( name="important_things", status="running", scheduler=mock.MagicMock(), last_success=None, ), dict( name="other_thing", status="enabled", scheduler=mock.MagicMock(), last_success="2012-01-23 10:23:23", ), ] def do_format(self): out = DisplayJobs().format(self.data) lines = out.split("\n") return lines def test_format(self): lines = self.do_format() assert len(lines) == 5 class TestDisplayActions(TestCase): @setup def setup_data(self): self.data = { "id": "something.23", "state": "UNKWN", "node": { "hostname": "something", "username": "a", }, "run_time": "sometime", "start_time": "sometime", "end_time": "sometime", "manual": False, "runs": [ dict( id="something.23.run_other_thing", state="unknown", start_time="2012-01-23 10:10:10.123456", end_time="", duration="", run_time="sometime", ), dict( id="something.1.run_foo", state="failed", start_time="2012-01-23 10:10:10.123456", end_time="2012-01-23 10:40:10.123456", duration="1234.123456", run_time="sometime", ), dict( id="something.23.run_other_thing", state="queued", start_time="2012-01-23 10:10:10.123456", end_time="", duration="", run_time="sometime", ), dict( id="something.42.weird_run", state="unknown", start_time=None, end_time=None, duration="", run_time=None, ), dict( id="something.43.skipped", state="skipped", start_time="2019-07-15 18:12:05", end_time="2019-07-16 01:31:50", duration="7:19:44.506211", ), dict( id="something.43.running", state="running", start_time="2019-07-15 18:12:05", end_time=None, duration="7:19:44.506211", ), ], } self.details = { "id": "something.1.foo", "state": "FAIL", "node": "localhost", "stdout": ["Blah", "blah", "blah"], "stderr": ["Crash", "and", "burn"], "command": "/bin/bash ./runme.sh now", "raw_command": "bash runme.sh now", "requirements": [".run_first_job"], } def format_lines(self): out = DisplayActionRuns().format(self.data) return out.split("\n") def test_format(self): lines = self.format_lines() assert len(lines) == 16, "\n".join(lines) class TestAddColorForState(TestCase): @setup_teardown def enable_color(self): with display.Color.enable(): yield def test_add_red(self): text = display.add_color_for_state(actionrun.ActionRun.FAILED) assert text.startswith(display.Color.colors["red"]), text def test_add_green(self): text = display.add_color_for_state(actionrun.ActionRun.RUNNING) assert text.startswith(display.Color.colors["green"]), text def test_add_blue(self): text = display.add_color_for_state(job.Job.STATUS_DISABLED) assert text.startswith(display.Color.colors["blue"]), text class TestDisplayNode(TestCase): node_source = { "name": "name", "hostname": "hostname", "username": "username", } def test_display_node(self): result = display.display_node(self.node_source) assert result == "username@hostname" def test_display_node_pool(self): source = {"name": "name", "nodes": [self.node_source]} result = display.display_node_pool(source) assert result == "name (1 node(s))" ================================================ FILE: tests/commands/retry_test.py ================================================ import random from unittest import mock import pytest from tron.commands import client from tron.commands import retry async def _empty_coro(*args, **kwargs): return None @pytest.fixture(autouse=True) def mock_sleep(): with mock.patch("asyncio.sleep", _empty_coro, autospec=None): yield @pytest.fixture(autouse=True) def mock_client(): with mock.patch.object(client, "Client", autospec=True) as m: m.return_value.url_base = "http://localhost" yield m @pytest.fixture(autouse=True) def mock_urlopen(): # prevent any requests from being made with mock.patch("urllib.request.urlopen", autospec=True) as m: yield m @pytest.fixture def mock_client_request(): with mock.patch.object(client, "request", autospec=True) as m: m.return_value = mock.Mock(error=False, content={}) # response yield m @mock.patch.object( client, "get_object_type_from_identifier", return_value=client.TronObjectIdentifier("JOB_RUN", "/a_job_run"), autospec=True, ) def test_retry_action_init_not_an_action(mock_get_obj_type, mock_client): tron_client = mock_client.return_value with pytest.raises(ValueError): retry.RetryAction(tron_client, "a_fake_action_run") @pytest.fixture def fake_retry_action(mock_client): tron_client = mock_client.return_value tron_client.action_runs.return_value = dict( action_name="a_fake_action", requirements=["required_action_0", "required_action_1"], triggered_by="a_fake_trigger_0 (done), a_fake_trigger_1", ) tron_client.job_runs.return_value = dict( job_name="a_fake_job", run_num=1234, runs=[ dict(action_name="required_action_0", state="succeeded"), dict(action_name="non_required_action", state="succeeded"), dict(action_name="required_action_1", state="failed"), dict(action_name="upstream_action_0", trigger_downstreams="a_fake_trigger_0"), dict(action_name="upstream_action_1", trigger_downstreams="a_fake_trigger_1"), tron_client.action_runs.return_value, ], ) with mock.patch.object( client, "get_object_type_from_identifier", side_effect=[ client.TronObjectIdentifier("ACTION_RUN", "/a_fake_job/0/a_fake_action"), client.TronObjectIdentifier("JOB_RUN", "/a_fake_job/0"), ], autospec=True, ): yield retry.RetryAction(tron_client, "a_fake_job.0.a_fake_action", use_latest_command=True) def test_retry_action_init_ok(fake_retry_action): assert fake_retry_action.retry_params == dict(command="retry", use_latest_command=1) assert fake_retry_action.full_action_name == "a_fake_job.0.a_fake_action" fake_retry_action.tron_client.action_runs.assert_called_once_with( "/a_fake_job/0/a_fake_action", num_lines=0, ) assert fake_retry_action.action_name == "a_fake_action" assert fake_retry_action.action_run_id.url == "/a_fake_job/0/a_fake_action" fake_retry_action.tron_client.job_runs.assert_called_once_with("/a_fake_job/0") assert fake_retry_action.job_run_name == "a_fake_job.0" assert fake_retry_action.job_run_id.url == "/a_fake_job/0" assert fake_retry_action._required_action_indices == {"required_action_0": 0, "required_action_1": 2} def test_check_trigger_statuses(fake_retry_action, event_loop): expected = dict(a_fake_trigger_0=True, a_fake_trigger_1=False) assert expected == event_loop.run_until_complete(fake_retry_action.check_trigger_statuses()) assert fake_retry_action.tron_client.action_runs.call_args_list[1] == mock.call( # 0th call is in init "/a_fake_job/0/a_fake_action", num_lines=0, ) def test_check_required_actions_statuses(fake_retry_action, event_loop): expected = dict(required_action_0=True, required_action_1=False) assert expected == event_loop.run_until_complete(fake_retry_action.check_required_actions_statuses()) assert fake_retry_action.tron_client.job_runs.call_args_list[1] == mock.call("/a_fake_job/0") # 0th call is in init @pytest.mark.parametrize( "expected,triggered_by,required_action_1_state", [ (False, "a_fake_trigger_0 (done), a_fake_trigger_1", "skipped"), # unpublished triggers (False, "a_fake_trigger_0 (done), a_fake_trigger_1 (done)", "failed"), # required not succeeded (True, "a_fake_trigger_0 (done), a_fake_trigger_1 (done)", "succeeded"), # all done ], ) def test_can_retry(fake_retry_action, event_loop, expected, triggered_by, required_action_1_state): fake_retry_action.tron_client.action_runs.return_value["triggered_by"] = triggered_by fake_retry_action.tron_client.job_runs.return_value["runs"][2]["state"] = required_action_1_state assert expected == event_loop.run_until_complete(fake_retry_action.can_retry()) def test_wait_for_deps_timeout(fake_retry_action, event_loop): assert not event_loop.run_until_complete(fake_retry_action.wait_for_deps(deps_timeout_s=3, poll_interval_s=1)) assert fake_retry_action._elapsed.seconds == 3 assert fake_retry_action.tron_client.action_runs.call_count == 5 # 1 in init, 4 in this test def test_wait_for_deps_all_deps_done(fake_retry_action, event_loop): fake_retry_action.tron_client.job_runs.return_value["runs"][2]["state"] = "skipped" fake_retry_action.tron_client.action_runs.return_value = None triggered_by_results = [ "a_fake_trigger_0 (done), a_fake_trigger_1", "a_fake_trigger_0 (done), a_fake_trigger_1", "a_fake_trigger_0 (done), a_fake_trigger_1 (done)", ] fake_retry_action.tron_client.action_runs.side_effect = [ dict( action_name="a_fake_action", requirements=["required_action_0", "required_action_1"], triggered_by=r, ) for r in triggered_by_results ] assert event_loop.run_until_complete(fake_retry_action.wait_for_deps(deps_timeout_s=3, poll_interval_s=1)) # 3rd triggered_by result returned on check at 2nd second assert fake_retry_action._elapsed.seconds == 2 assert fake_retry_action.tron_client.action_runs.call_count == 4 # 1 in init, 3 in this test @pytest.mark.parametrize("expected,error", [(False, True), (True, False)]) def test_issue_retry(fake_retry_action, mock_client_request, event_loop, expected, error): mock_client_request.return_value.error = error assert expected == event_loop.run_until_complete(fake_retry_action.issue_retry()) assert expected == fake_retry_action.succeeded def test_wait_for_retry_deps_not_done(fake_retry_action, mock_client_request, event_loop): assert not event_loop.run_until_complete( fake_retry_action.wait_and_retry(deps_timeout_s=10, poll_interval_s=1, jitter=True), ) assert fake_retry_action._elapsed.seconds == 10 # timeout mock_client_request.assert_not_called() # retry not attempted def test_wait_for_retry_deps_done(fake_retry_action, mock_client_request, event_loop): fake_retry_action.tron_client.job_runs.return_value["runs"][2]["state"] = "skipped" fake_retry_action.tron_client.action_runs.return_value[ "triggered_by" ] = "a_fake_trigger_0 (done), a_fake_trigger_1 (done)" mock_client_request.return_value.error = False random.seed(1) # init delay is 1s assert event_loop.run_until_complete( fake_retry_action.wait_and_retry(deps_timeout_s=10, poll_interval_s=5, jitter=True), ) assert fake_retry_action._elapsed.seconds == 1 # init delay only mock_client_request.assert_called_once_with( "http://localhost/a_fake_job/0/a_fake_action", data=dict(command="retry", use_latest_command=1), user_attribution=True, ) @mock.patch.object(retry, "RetryAction", autospec=True) def test_retry_actions(mock_retry_action, mock_client, event_loop): mock_wait_and_retry = mock_retry_action.return_value.wait_and_retry mock_wait_and_retry.return_value = _empty_coro() r_actions = retry.retry_actions( "http://localhost", ["a_job.0.an_action_0", "another_job.1.an_action_1"], use_latest_command=True, deps_timeout_s=4, ) assert r_actions == [mock_retry_action.return_value] * 2 assert mock_retry_action.call_args_list == [ mock.call(mock_client.return_value, "a_job.0.an_action_0", use_latest_command=True), mock.call(mock_client.return_value, "another_job.1.an_action_1", use_latest_command=True), ] assert mock_wait_and_retry.call_args_list == [ mock.call(deps_timeout_s=4, jitter=False), mock.call(deps_timeout_s=4), ] ================================================ FILE: tests/config/__init__.py ================================================ ================================================ FILE: tests/config/config_parse_test.py ================================================ import datetime import os import shutil import tempfile from unittest import mock import pytest import pytz from testifycompat import assert_equal from testifycompat import assert_in from testifycompat import run from testifycompat import setup from testifycompat import teardown from testifycompat import TestCase from tests.assertions import assert_raises from tron.config import config_parse from tron.config import config_utils from tron.config import ConfigError from tron.config import schedule_parse from tron.config import schema from tron.config.config_parse import build_format_string_validator from tron.config.config_parse import CLEANUP_ACTION_NAME from tron.config.config_parse import valid_cleanup_action_name from tron.config.config_parse import valid_config from tron.config.config_parse import valid_job from tron.config.config_parse import valid_node_pool from tron.config.config_parse import valid_output_stream_dir from tron.config.config_parse import validate_fragment from tron.config.config_utils import NullConfigContext from tron.config.schedule_parse import ConfigDailyScheduler from tron.config.schema import ConfigNodeAffinity from tron.config.schema import MASTER_NAMESPACE BASE_CONFIG = dict( ssh_options=dict(agent=False, identities=["tests/test_id_rsa"]), time_zone="EST", output_stream_dir="/tmp", nodes=[ dict(name="node0", hostname="node0"), dict(name="node1", hostname="node1"), ], node_pools=[dict(name="NodePool", nodes=["node0", "node1"])], ) def make_ssh_options(): return schema.ConfigSSHOptions( agent=False, identities=("tests/test_id_rsa",), known_hosts_file=None, connect_timeout=30, idle_connection_timeout=3600, jitter_min_load=4, jitter_max_delay=20, jitter_load_factor=1, ) def make_mock_schedule(): return ConfigDailyScheduler( days=set(), hour=0, minute=0, second=0, original="00:00:00 ", jitter=None, ) def make_command_context(): return { "python": "/usr/bin/python", "batch_dir": "/tron/batch/test/foo", } def make_nodes(): return { "node0": schema.ConfigNode( name="node0", username="foo", hostname="node0", port=22, ), "node1": schema.ConfigNode( name="node1", username="foo", hostname="node1", port=22, ), } def make_node_pools(): return { "NodePool": schema.ConfigNodePool( nodes=("node0", "node1"), name="NodePool", ), } def make_mesos_options(): return schema.ConfigMesos( master_address=None, master_port=5050, secret_file=None, role="*", principal="tron", enabled=False, default_volumes=(), dockercfg_location=None, offer_timeout=300, ) def make_k8s_options(): return schema.ConfigKubernetes(enabled=False, non_retryable_exit_codes=(), default_volumes=()) def make_action(**kwargs): kwargs.setdefault("name", "action"), kwargs.setdefault("command", "command") kwargs.setdefault("executor", schema.ExecutorTypes.ssh.value) kwargs.setdefault("requires", ()) kwargs.setdefault("expected_runtime", datetime.timedelta(1)) return schema.ConfigAction(**kwargs) def make_cleanup_action(**kwargs): kwargs.setdefault("name", "cleanup"), kwargs.setdefault("command", "command") kwargs.setdefault("executor", schema.ExecutorTypes.ssh.value) kwargs.setdefault("expected_runtime", datetime.timedelta(1)) return schema.ConfigCleanupAction(**kwargs) def make_job(**kwargs): kwargs.setdefault("namespace", "MASTER") kwargs.setdefault("name", f"{kwargs['namespace']}.job_name") kwargs.setdefault("node", "node0") kwargs.setdefault("enabled", True) kwargs.setdefault("monitoring", {}) kwargs.setdefault( "schedule", schedule_parse.ConfigDailyScheduler( days=set(), hour=16, minute=30, second=0, original="16:30:00 ", jitter=None, ), ) kwargs.setdefault("actions", {"action": make_action()}) kwargs.setdefault("queueing", True) kwargs.setdefault("run_limit", 50) kwargs.setdefault("all_nodes", False) kwargs.setdefault("cleanup_action", make_cleanup_action()) kwargs.setdefault("max_runtime") kwargs.setdefault("allow_overlap", False) kwargs.setdefault("time_zone", None) kwargs.setdefault("expected_runtime", datetime.timedelta(0, 3600)) kwargs.setdefault("use_k8s", False) return schema.ConfigJob(**kwargs) def make_master_jobs(): return { "MASTER.test_job0": make_job( name="MASTER.test_job0", schedule=make_mock_schedule(), expected_runtime=datetime.timedelta(1), ), "MASTER.test_job1": make_job( name="MASTER.test_job1", schedule=schedule_parse.ConfigDailyScheduler( days={1, 3, 5}, hour=0, minute=30, second=0, original="00:30:00 MWF", jitter=None, ), actions={ "action": make_action( requires=("action1",), expected_runtime=datetime.timedelta(0, 7200), ), "action1": make_action( name="action1", expected_runtime=datetime.timedelta(0, 7200), ), }, time_zone=pytz.timezone("Pacific/Auckland"), expected_runtime=datetime.timedelta(1), cleanup_action=None, allow_overlap=True, ), "MASTER.test_job2": make_job( name="MASTER.test_job2", node="node1", actions={ "action2_0": make_action( name="action2_0", command="test_command2.0", ), }, time_zone=pytz.timezone("Pacific/Auckland"), expected_runtime=datetime.timedelta(1), cleanup_action=None, ), "MASTER.test_job_actions_dict": make_job( name="MASTER.test_job_actions_dict", node="node1", schedule=make_mock_schedule(), actions={ "action": make_action(), "action1": make_action(name="action1"), "action2": make_action( name="action2", requires=("action", "action1"), node="node0", ), }, cleanup_action=None, expected_runtime=datetime.timedelta(1), ), "MASTER.test_job4": make_job( name="MASTER.test_job4", node="NodePool", schedule=schedule_parse.ConfigDailyScheduler( original="00:00:00 ", hour=0, minute=0, second=0, days=set(), jitter=None, ), all_nodes=True, enabled=False, cleanup_action=None, expected_runtime=datetime.timedelta(1), ), "MASTER.test_job_mesos": make_job( name="MASTER.test_job_mesos", node="NodePool", schedule=schedule_parse.ConfigDailyScheduler( original="00:00:00 ", hour=0, minute=0, second=0, days=set(), jitter=None, ), actions={ "action_mesos": make_action( name="action_mesos", command="test_command_mesos", executor=schema.ExecutorTypes.mesos.value, cpus=0.1, mem=100, disk=600, docker_image="container:latest", ), }, cleanup_action=None, expected_runtime=datetime.timedelta(1), ), "MASTER.test_job_k8s": make_job( name="MASTER.test_job_k8s", node="NodePool", schedule=schedule_parse.ConfigDailyScheduler( original="00:00:00 ", hour=0, minute=0, second=0, days=set(), jitter=None, ), actions={ "action_k8s": make_action( name="action_k8s", command="test_command_k8s", executor=schema.ExecutorTypes.kubernetes.value, cpus=0.1, mem=100, disk=600, cap_add=["KILL"], cap_drop=["CHOWN", "KILL"], docker_image="container:latest", secret_env=dict( TEST_SECRET=schema.ConfigSecretSource(secret_name="tron-secret-test-secret--1", key="secret_1") ), secret_volumes=( schema.ConfigSecretVolume( secret_volume_name="abc", secret_name="secret1", container_path="/b/c", default_mode="0644", items=(schema.ConfigSecretVolumeItem(key="secret1", path="abcd", mode="777"),), ), ), projected_sa_volumes=( schema.ConfigProjectedSAVolume( container_path="/var/secrets/whatever", audience="foo.bar", expiration_seconds=1800, ), ), node_selectors={"yelp.com/pool": "default"}, node_affinities=( ConfigNodeAffinity( key="instance_type", operator="In", value=("a1.1xlarge",), ), ), ), }, cleanup_action=None, expected_runtime=datetime.timedelta(1), ), } def make_tron_config( action_runner=None, output_stream_dir="/tmp", command_context=None, ssh_options=None, time_zone=pytz.timezone("EST"), state_persistence=config_parse.DEFAULT_STATE_PERSISTENCE, nodes=None, node_pools=None, jobs=None, mesos_options=None, k8s_options=None, read_json=False, ): return schema.TronConfig( action_runner=action_runner or {}, output_stream_dir=output_stream_dir, command_context=command_context or dict(batch_dir="/tron/batch/test/foo", python="/usr/bin/python"), ssh_options=ssh_options or make_ssh_options(), time_zone=time_zone, state_persistence=state_persistence, nodes=nodes or make_nodes(), node_pools=node_pools or make_node_pools(), jobs=jobs or make_master_jobs(), mesos_options=mesos_options or make_mesos_options(), k8s_options=k8s_options or make_k8s_options(), read_json=read_json, ) def make_named_tron_config(jobs=None): return schema.NamedTronConfig(jobs=jobs or make_master_jobs()) class ConfigTestCase(TestCase): JOBS_CONFIG = dict( jobs=[ dict( name="test_job0", node="node0", schedule="daily 00:00:00", actions=[dict(name="action", command="command")], cleanup_action=dict(command="command"), ), dict( name="test_job1", node="node0", schedule="daily 00:30:00 MWF", allow_overlap=True, time_zone="Pacific/Auckland", actions=[ dict( name="action", command="command", requires=["action1"], expected_runtime="2h", ), dict( name="action1", command="command", expected_runtime="2h", ), ], ), dict( name="test_job2", node="node1", schedule="daily 16:30:00", expected_runtime="1d", time_zone="Pacific/Auckland", actions=[dict(name="action2_0", command="test_command2.0")], ), dict( name="test_job_actions_dict", node="node1", schedule="daily 00:00:00 ", actions=dict( action=dict(command="command"), action1=dict(command="command"), action2=dict( node="node0", command="command", requires=["action", "action1"], ), ), ), dict( name="test_job4", node="NodePool", all_nodes=True, schedule="daily", enabled=False, actions=[dict(name="action", command="command")], ), dict( name="test_job_mesos", node="NodePool", schedule="daily", actions=[ dict( name="action_mesos", executor="mesos", command="test_command_mesos", cpus=0.1, mem=100, disk=600, docker_image="container:latest", ), ], ), dict( name="test_job_k8s", node="NodePool", schedule="daily", actions=[ dict( name="action_k8s", executor="kubernetes", command="test_command_k8s", cpus=0.1, mem=100, disk=600, docker_image="container:latest", secret_env=dict(TEST_SECRET=dict(secret_name="tron-secret-test-secret--1", key="secret_1")), secret_volumes=[ dict( secret_volume_name="abc", secret_name="secret1", container_path="/b/c", default_mode="0644", items=[ dict(key="secret1", path="abcd", mode="777"), ], ), ], projected_sa_volumes=[ dict( container_path="/var/secrets/whatever", audience="foo.bar", expiration_seconds=1800, ), ], cap_add=["KILL"], cap_drop=["CHOWN", "KILL"], node_selectors={"yelp.com/pool": "default"}, node_affinities=[{"key": "instance_type", "operator": "In", "value": ["a1.1xlarge"]}], ), ], ), ], ) config = dict( command_context=dict( batch_dir="/tron/batch/test/foo", python="/usr/bin/python", ), **BASE_CONFIG, **JOBS_CONFIG, ) @mock.patch.dict("tron.config.config_parse.ValidateNode.defaults") def test_attributes(self): config_parse.ValidateNode.defaults["username"] = "foo" expected = make_tron_config() test_config = valid_config(self.config) assert test_config.command_context == expected.command_context assert test_config.ssh_options == expected.ssh_options assert test_config.mesos_options == expected.mesos_options assert test_config.time_zone == expected.time_zone assert test_config.nodes == expected.nodes assert test_config.node_pools == expected.node_pools assert test_config.k8s_options == expected.k8s_options assert test_config.read_json == expected.read_json for key in ["0", "1", "2", "_actions_dict", "4", "_mesos"]: job_name = f"MASTER.test_job{key}" assert job_name in test_config.jobs, f"{job_name} in test_config.jobs" assert job_name in expected.jobs, f"{job_name} in test_config.jobs" assert test_config.jobs[job_name] == expected.jobs[job_name] assert test_config == expected def test_empty_node_test(self): valid_config(dict(nodes=None)) class TestNamedConfig(TestCase): config = ConfigTestCase.JOBS_CONFIG def test_attributes(self): expected = make_named_tron_config( jobs={ "test_job": make_job( name="test_job", namespace="test_namespace", schedule=make_mock_schedule(), expected_runtime=datetime.timedelta(1), ), }, ) test_config = validate_fragment( "test_namespace", dict( jobs=[ dict( name="test_job", namespace="test_namespace", node="node0", schedule="daily 00:00:00 ", actions=[dict(name="action", command="command")], cleanup_action=dict(command="command"), ), ], ), ) assert_equal(test_config, expected) def test_attributes_with_master_context(self): expected = make_named_tron_config( jobs={ "test_namespace.test_job": make_job( name="test_namespace.test_job", namespace="test_namespace", schedule=make_mock_schedule(), expected_runtime=datetime.timedelta(1), ), }, ) master_config = dict( nodes=[ dict( name="node0", hostname="node0", ), ], node_pools=[ dict( name="nodepool0", nodes=["node0"], ), ], ) test_config = validate_fragment( "test_namespace", dict( jobs=[ dict( name="test_job", namespace="test_namespace", node="node0", schedule="daily 00:00:00", actions=[dict(name="action", command="command")], cleanup_action=dict(command="command"), ), ], ), master_config=master_config, ) assert_equal(test_config, expected) def test_invalid_job_node_with_master_context(self): master_config = dict( nodes=[ dict( name="node0", hostname="node0", ), ], ) test_config = dict( jobs=[ dict( name="test_job", namespace="test_namespace", node="node1", schedule="daily 00:30:00 ", actions=[dict(name="action", command="command")], cleanup_action=dict(command="command"), ), ], ) expected_message = "Unknown node name node1 at test_namespace.NamedConfigFragment.jobs.Job.test_job.node" exception = assert_raises( ConfigError, validate_fragment, "test_namespace", test_config, master_config, ) assert_in(expected_message, str(exception)) def test_invalid_action_node_with_master_context(self): master_config = dict( nodes=[ dict( name="node0", hostname="node0", ), ], node_pools=[ dict( name="nodepool0", nodes=["node0"], ), ], ) test_config = dict( jobs=[ dict( name="test_job", namespace="test_namespace", node="node0", schedule="daily 00:30:00 ", actions=[dict(name="action", node="nodepool1", command="command")], cleanup_action=dict(command="command"), ), ], ) expected_message = "Unknown node name nodepool1 at test_namespace.NamedConfigFragment.jobs.Job.test_job.actions.Action.action.node" exception = assert_raises( ConfigError, validate_fragment, "test_namespace", test_config, master_config, ) assert_in(expected_message, str(exception)) class TestJobConfig(TestCase): def test_no_actions(self): test_config = dict( jobs=[ dict(name="test_job0", node="node0", schedule="daily 00:30:00 "), ], **BASE_CONFIG, ) expected_message = "Job test_job0 is missing options: actions" exception = assert_raises( ConfigError, valid_config, test_config, ) assert_in(expected_message, str(exception)) def test_empty_actions(self): test_config = dict( jobs=[ dict( name="test_job0", node="node0", schedule="daily 00:30:00 ", actions=None, ), ], **BASE_CONFIG, ) expected_message = "Value at config.jobs.Job.test_job0.actions" exception = assert_raises( ConfigError, valid_config, test_config, ) assert_in(expected_message, str(exception)) def test_dupe_names(self): test_config = dict( jobs=[ dict( name="test_job0", node="node0", schedule="daily 00:30:00", actions=[ dict(name="action", command="cmd"), dict(name="action", command="cmd"), ], ), ], **BASE_CONFIG, ) expected = "Duplicate name action at config.jobs.Job.test_job0.actions" exception = assert_raises( ConfigError, valid_config, test_config, ) assert_in(expected, str(exception)) def test_bad_requires(self): test_config = dict( jobs=[ dict( name="test_job0", node="node0", schedule="daily 00:30:00", actions=[dict(name="action", command="cmd")], ), dict( name="test_job1", node="node0", schedule="daily 00:30:00", actions=[ dict( name="action1", command="cmd", requires=["action"], ), ], ), ], **BASE_CONFIG, ) expected_message = "jobs.MASTER.test_job1.action1 has a dependency " '"action" that is not in the same job!' exception = assert_raises( ConfigError, valid_config, test_config, ) assert_in(expected_message, str(exception)) def test_circular_dependency(self): test_config = dict( jobs=[ dict( name="test_job0", node="node0", schedule="daily 00:30:00", actions=[ dict( name="action1", command="cmd", requires=["action2"], ), dict( name="action2", command="cmd", requires=["action1"], ), ], ), ], **BASE_CONFIG, ) expect = "Circular dependency in job.MASTER.test_job0: action1 -> action2" exception = assert_raises( ConfigError, valid_config, test_config, ) assert_in(expect, str(exception)) def test_circular_dependency_multiaction(self): test_config = dict( jobs=[ dict( name="test_job0", node="node0", schedule="daily 00:30:00", actions=[ dict( name="action1", command="cmd", requires=["action2"], ), dict( name="action2", command="cmd", requires=["action3"], ), dict( name="action3", command="cmd", requires=["action4"], ), dict( name="action4", command="cmd", requires=["action5"], ), dict( name="action5", command="cmd", requires=["action3"], ), ], ), ], **BASE_CONFIG, ) expect = "Circular dependency in job.MASTER.test_job0: action3 -> action4 -> action5" exception = assert_raises( ConfigError, valid_config, test_config, ) assert_in(expect, str(exception)) def test_config_cleanup_name_collision(self): test_config = dict( jobs=[ dict( name="test_job0", node="node0", schedule="daily 00:30:00", actions=[ dict(name=CLEANUP_ACTION_NAME, command="cmd"), ], ), ], **BASE_CONFIG, ) expected_message = "config.jobs.Job.test_job0.actions.Action.cleanup.name" exception = assert_raises( ConfigError, valid_config, test_config, ) assert_in(expected_message, str(exception)) def test_config_cleanup_action_name(self): test_config = dict( jobs=[ dict( name="test_job0", node="node0", schedule="daily 00:30:00", actions=[ dict(name="action", command="cmd"), ], cleanup_action=dict(name="gerald", command="cmd"), ), ], **BASE_CONFIG, ) expected_msg = "Cleanup actions cannot have custom names" exception = assert_raises( ConfigError, valid_config, test_config, ) assert_in(expected_msg, str(exception)) def test_config_cleanup_requires(self): test_config = dict( jobs=[ dict( name="test_job0", node="node0", schedule="daily 00:30:00", actions=[ dict(name="action", command="cmd"), ], cleanup_action=dict(command="cmd", requires=["action"]), ), ], **BASE_CONFIG, ) expected_msg = "Unknown keys in CleanupAction : requires" exception = assert_raises( ConfigError, valid_config, test_config, ) assert_equal(expected_msg, str(exception)) def test_validate_job_no_actions(self): job_config = dict( name="job_name", node="localhost", schedule="daily 00:30:00", actions=[], ) config_context = config_utils.ConfigContext( "config", ["localhost"], None, None, ) expected_msg = "Required non-empty list at config.Job.job_name.actions" exception = assert_raises( ConfigError, valid_job, job_config, config_context, ) assert_in(expected_msg, str(exception)) class TestValidSecretSource(TestCase): def test_missing_secret_name(self): secret_env = dict(key="no_secret_name") with pytest.raises(ConfigError) as missing_exc: config_parse.valid_secret_source(secret_env, NullConfigContext) assert "missing options: secret" in str(missing_exc.value) def test_validate_job_extra_secret_env(self): secret_env = dict( secret_name="tron-secret-k8s-name-no--secret--name", key="no_secret_name", extra_key="unknown", ) with pytest.raises(ConfigError) as missing_exc: config_parse.valid_secret_source(secret_env, NullConfigContext) assert "Unknown keys in SecretSource : extra_key" in str(missing_exc.value) def test_valid_job_secret_env_success(self): secret_env = dict( secret_name="tron-secret-k8s-name-no--secret--name", key="no_secret_name", ) expected_env = schema.ConfigSecretSource(**secret_env) built_env = config_parse.valid_secret_source(secret_env, NullConfigContext) assert built_env == expected_env class TestNodeConfig(TestCase): def test_validate_node_pool(self): config_node_pool = valid_node_pool( dict(name="theName", nodes=["node1", "node2"]), ) assert_equal(config_node_pool.name, "theName") assert_equal(len(config_node_pool.nodes), 2) def test_overlap_node_and_node_pools(self): tron_config = dict( nodes=[ dict(name="sameName", hostname="localhost"), ], node_pools=[ dict(name="sameName", nodes=["sameNode"]), ], ) expected_msg = "Node and NodePool names must be unique sameName" exception = assert_raises(ConfigError, valid_config, tron_config) assert_in(expected_msg, str(exception)) def test_invalid_node_name(self): test_config = dict( jobs=[ dict( name="test_job0", node="unknown_node", schedule="daily 00:30:00", actions=[dict(name="action", command="cmd")], ), ], **BASE_CONFIG, ) expected_msg = "Unknown node name unknown_node at config.jobs.Job.test_job0.node" exception = assert_raises( ConfigError, valid_config, test_config, ) assert_equal(expected_msg, str(exception)) def test_invalid_nested_node_pools(self): test_config = dict( nodes=[ dict(name="node0", hostname="node0"), dict(name="node1", hostname="node1"), ], node_pools=[ dict(name="pool0", nodes=["node1"]), dict(name="pool1", nodes=["node0", "pool0"]), ], jobs=[ dict( name="test_job0", node="pool1", schedule="daily 00:30:00", actions=[dict(name="action", command="cmd")], ), ], ) expected_msg = "NodePool pool1 contains other NodePools: pool0" exception = assert_raises( ConfigError, valid_config, test_config, ) assert_in(expected_msg, str(exception)) def test_invalid_node_pool_config(self): test_config = dict( nodes=[ dict(name="node0", hostname="node0"), dict(name="node1", hostname="node1"), ], node_pools=[ dict(name="pool0", hostname=["node1"]), dict(name="pool1", nodes=["node0", "pool0"]), ], jobs=[ dict( name="test_job0", node="pool1", schedule="daily 00:30:00", actions=[dict(name="action", command="cmd")], ), ], ) expected_msg = "NodePool pool0 is missing options" exception = assert_raises( ConfigError, valid_config, test_config, ) assert_in(expected_msg, str(exception)) def test_invalid_named_update(self): test_config = dict(bozray=None) expected_message = "Unknown keys in NamedConfigFragment : bozray" exception = assert_raises( ConfigError, validate_fragment, "foo", test_config, ) assert_in(expected_message, str(exception)) class TestValidateJobs(TestCase): def test_valid_jobs_success(self): test_config = dict( jobs=[ dict( name="test_job0", node="node0", schedule="daily", expected_runtime="20m", actions=[ dict( name="action", command="command", expected_runtime="20m", ), dict( name="action_mesos", command="command", executor="mesos", cpus=4, mem=300, disk=600, constraints=[ dict( attribute="pool", operator="LIKE", value="default", ), ], docker_image="my_container:latest", docker_parameters=[ dict(key="label", value="labelA"), dict(key="label", value="labelB"), ], env=dict(USER="batch"), extra_volumes=[ dict( container_path="/tmp", host_path="/home/tmp", mode="RO", ), ], ), dict( name="test_trigger_attrs", command="foo", triggered_by=["foo.bar"], trigger_downstreams=True, ), ], cleanup_action=dict(command="command"), ), ], **BASE_CONFIG, ) expected_jobs = { "MASTER.test_job0": make_job( name="MASTER.test_job0", schedule=make_mock_schedule(), actions={ "action": make_action( expected_runtime=datetime.timedelta(0, 1200), ), "action_mesos": make_action( name="action_mesos", executor=schema.ExecutorTypes.mesos.value, cpus=4.0, mem=300.0, disk=600.0, constraints=( schema.ConfigConstraint( attribute="pool", operator="LIKE", value="default", ), ), docker_image="my_container:latest", docker_parameters=( schema.ConfigParameter( key="label", value="labelA", ), schema.ConfigParameter( key="label", value="labelB", ), ), env={"USER": "batch"}, extra_volumes=( schema.ConfigVolume( container_path="/tmp", host_path="/home/tmp", mode=schema.VolumeModes.RO.value, ), ), expected_runtime=datetime.timedelta(hours=24), ), "test_trigger_attrs": make_action( name="test_trigger_attrs", command="foo", triggered_by=("foo.bar",), trigger_downstreams=True, ), }, expected_runtime=datetime.timedelta(0, 1200), ), } context = config_utils.ConfigContext( "config", ["node0"], None, MASTER_NAMESPACE, ) config_parse.validate_jobs(test_config, context) assert expected_jobs == test_config["jobs"] class TestValidCleanupActionName(TestCase): def test_valid_cleanup_action_name_pass(self): name = valid_cleanup_action_name(CLEANUP_ACTION_NAME, None) assert_equal(CLEANUP_ACTION_NAME, name) def test_valid_cleanup_action_name_fail(self): assert_raises( ConfigError, valid_cleanup_action_name, "other", NullConfigContext, ) class TestValidOutputStreamDir(TestCase): @setup def setup_dir(self): self.dir = tempfile.mkdtemp() @teardown def teardown_dir(self): shutil.rmtree(self.dir) def test_valid_dir(self): path = valid_output_stream_dir(self.dir, NullConfigContext) assert_equal(self.dir, path) def test_missing_dir(self): exception = assert_raises( ConfigError, valid_output_stream_dir, "bogus-dir", NullConfigContext, ) assert_in("is not a directory", str(exception)) # TODO: docker tests run as root so everything is writeable # def test_no_ro_dir(self): # os.chmod(self.dir, stat.S_IRUSR) # exception = assert_raises( # ConfigError, # valid_output_stream_dir, self.dir, NullConfigContext, # ) # assert_in("is not writable", str(exception)) def test_missing_with_partial_context(self): dir = "/bogus/path/does/not/exist" context = config_utils.PartialConfigContext("path", "MASTER") path = config_parse.valid_output_stream_dir(dir, context) assert_equal(path, dir) class TestBuildFormatStringValidator(TestCase): @setup def setup_keys(self): self.context = dict.fromkeys(["one", "seven", "stars"]) self.validator = build_format_string_validator(self.context) def test_validator_passes(self): template = "The {one} thing I {seven} is {stars}" assert self.validator(template, NullConfigContext) def test_validator_unknown_variable_error(self): template = "The {one} thing I {seven} is {unknown}" exception = assert_raises( ConfigError, self.validator, template, NullConfigContext, ) assert_in("Unknown context variable", str(exception)) def test_validator_passes_with_context(self): template = "The {one} thing I {seven} is {mars}" context = config_utils.ConfigContext( None, None, {"mars": "ok"}, None, ) assert self.validator(template, context) == template def test_validator_valid_string_without_no_percent_escape(self): template = "The {one} {seven} thing is {mars} --year %Y" context = config_utils.ConfigContext( path=None, nodes=None, command_context={"mars": "ok"}, namespace=None, ) assert self.validator(template, context) class TestValidateConfigMapping(TestCase): config = dict(**BASE_CONFIG, command_context=dict(some_var="The string")) def test_validate_config_mapping_missing_master(self): config_mapping = {"other": mock.Mock()} seq = config_parse.validate_config_mapping(config_mapping) exception = assert_raises(ConfigError, list, seq) assert_in("requires a MASTER namespace", str(exception)) def test_validate_config_mapping(self): master_config = self.config other_config = TestNamedConfig.config config_mapping = { "other": other_config, MASTER_NAMESPACE: master_config, } result = list(config_parse.validate_config_mapping(config_mapping)) assert_equal(len(result), 2) assert_equal(result[0][0], MASTER_NAMESPACE) assert_equal(result[1][0], "other") class TestConfigContainer(TestCase): config = BASE_CONFIG @setup def setup_container(self): other_config = TestNamedConfig.config self.config_mapping = { MASTER_NAMESPACE: valid_config(self.config), "other": validate_fragment("other", other_config), } self.container = config_parse.ConfigContainer(self.config_mapping) def test_create(self): config_mapping = { MASTER_NAMESPACE: self.config, "other": TestNamedConfig.config, } container = config_parse.ConfigContainer.create(config_mapping) assert_equal(set(container.configs.keys()), {"MASTER", "other"}) def test_create_missing_master(self): config_mapping = {"other": mock.Mock()} assert_raises( ConfigError, config_parse.ConfigContainer.create, config_mapping, ) def test_get_job_names(self): job_names = self.container.get_job_names() expected = [ "test_job1", "test_job0", "test_job_actions_dict", "test_job2", "test_job4", "test_job_mesos", "test_job_k8s", ] assert_equal(set(job_names), set(expected)) def test_get_jobs(self): expected = [ "test_job1", "test_job0", "test_job_actions_dict", "test_job2", "test_job4", "test_job_mesos", "test_job_k8s", ] assert_equal(set(expected), set(self.container.get_jobs().keys())) def test_get_node_names(self): node_names = self.container.get_node_names() expected = {"node0", "node1", "NodePool"} assert_equal(node_names, expected) class TestValidateSSHOptions(TestCase): @setup def setup_context(self): self.context = config_utils.NullConfigContext self.config = {"agent": True, "identities": []} @mock.patch.dict("tron.config.config_parse.os.environ") def test_post_validation_failed(self): if "SSH_AUTH_SOCK" in os.environ: del os.environ["SSH_AUTH_SOCK"] assert_raises( ConfigError, config_parse.valid_ssh_options.validate, self.config, self.context, ) @mock.patch.dict("tron.config.config_parse.os.environ") def test_post_validation_success(self): os.environ["SSH_AUTH_SOCK"] = "something" config = config_parse.valid_ssh_options.validate( self.config, self.context, ) assert_equal(config.agent, True) class TestValidateIdentityFile(TestCase): @setup def setup_context(self): self.context = config_utils.NullConfigContext self.private_file = tempfile.NamedTemporaryFile() def test_valid_identity_file_missing_private_key(self): exception = assert_raises( ConfigError, config_parse.valid_identity_file, "/file/not/exist", self.context, ) assert_in("Private key file", str(exception)) def test_valid_identity_files_missing_public_key(self): filename = self.private_file.name exception = assert_raises( ConfigError, config_parse.valid_identity_file, filename, self.context, ) assert_in("Public key file", str(exception)) def test_valid_identity_files_valid(self): filename = self.private_file.name fh_private = open(filename + ".pub", "w") try: config = config_parse.valid_identity_file(filename, self.context) finally: fh_private.close() os.unlink(fh_private.name) assert_equal(config, filename) def test_valid_identity_files_missing_with_partial_context(self): path = "/bogus/file/does/not/exist" context = config_utils.PartialConfigContext("path", "MASTER") file_path = config_parse.valid_identity_file(path, context) assert_equal(path, file_path) class TestValidKnownHostsFile(TestCase): @setup def setup_context(self): self.context = config_utils.NullConfigContext self.known_hosts_file = tempfile.NamedTemporaryFile() def test_valid_known_hosts_file_exists(self): filename = config_parse.valid_known_hosts_file( self.known_hosts_file.name, self.context, ) assert_equal(filename, self.known_hosts_file.name) def test_valid_known_hosts_file_missing(self): exception = assert_raises( ConfigError, config_parse.valid_known_hosts_file, "/bogus/path", self.context, ) assert_in("Known hosts file /bogus/path", str(exception)) def test_valid_known_hosts_file_missing_partial_context(self): context = config_utils.PartialConfigContext expected = "/bogus/does/not/exist" filename = config_parse.valid_known_hosts_file( expected, context, ) assert_equal(filename, expected) class TestValidateVolume(TestCase): @setup def setup_context(self): self.context = config_utils.NullConfigContext def test_missing_container_path(self): config = { "container_path_typo": "/nail/srv", "host_path": "/tmp", "mode": "RO", } assert_raises( ConfigError, config_parse.valid_volume.validate, config, self.context, ) def test_missing_host_path(self): config = { "container_path": "/nail/srv", "hostPath": "/tmp", "mode": "RO", } assert_raises( ConfigError, config_parse.valid_volume.validate, config, self.context, ) def test_invalid_mode(self): config = { "container_path": "/nail/srv", "host_path": "/tmp", "mode": "RA", } assert_raises( ConfigError, config_parse.valid_volume.validate, config, self.context, ) def test_valid(self): config = { "container_path": "/nail/srv", "host_path": "/tmp", "mode": schema.VolumeModes.RO.value, } assert_equal( schema.ConfigVolume(**config), config_parse.valid_volume.validate(config, self.context), ) def test_mesos_default_volumes(self): mesos_options = {"master_address": "mesos_master"} mesos_options["default_volumes"] = [ { "container_path": "/nail/srv", "host_path": "/tmp", "mode": "RO", }, { "container_path": "/nail/srv", "host_path": "/tmp", "mode": "invalid", }, ] with pytest.raises(ConfigError): config_parse.valid_mesos_options.validate(mesos_options, self.context) # After we fix the error, expect error to go away. mesos_options["default_volumes"][1]["mode"] = "RW" assert config_parse.valid_mesos_options.validate( mesos_options, self.context, ) def test_k8s_default_volumes(self): k8s_options = {"kubeconfig_path": "some_path"} k8s_options["default_volumes"] = [ { "container_path": "/nail/srv", "host_path": "/tmp", "mode": "RO", }, { "container_path": "/nail/srv", "host_path": "/tmp", "mode": "invalid", }, ] with pytest.raises(ConfigError): config_parse.valid_kubernetes_options.validate(k8s_options, self.context) # After we fix the error, expect error to go away. k8s_options["default_volumes"][1]["mode"] = "RW" assert config_parse.valid_kubernetes_options.validate( k8s_options, self.context, ) class TestValidPermissionMode: @pytest.mark.parametrize( ("permission", "normalized"), [("777", "777"), ("0", "0"), ("0000", "0000"), ("0123", "0123"), (0, "0"), (7777, "7777")], ) def test_valid_permissions(self, permission, normalized): result = config_parse.valid_permission_mode(permission, NullConfigContext) assert result == normalized @pytest.mark.parametrize("permission", ["778", "é", -1, "", {}, [], ()]) def test_invalid_permissions(self, permission): with pytest.raises(ConfigError): config_parse.valid_permission_mode(permission, NullConfigContext) class TestValidSecretVolumeItem: @pytest.mark.parametrize( "config", [ {"path": "abc"}, { "key": "abc", }, { "key": "abc", "path": "abc", "extra_key": None, }, {"key": "abc", "path": "abc", "mode": "a"}, ], ) def test_invalid(self, config): with pytest.raises(ConfigError): config_parse.valid_secret_volume_item(config, NullConfigContext) @pytest.mark.parametrize( "config", [{"key": "abc", "path": "abc"}, {"key": "abc", "path": "abc", "mode": "777"}], ) def test_valid_job_secret_volume_success(self, config): config_parse.valid_secret_volume_item(config, NullConfigContext) @pytest.mark.parametrize( "item_config, default_mode, expected", [ ( # Item inherits volume default_mode {"key": "s1", "path": "p1"}, "0755", "0755", ), ( # Item explicit mode overrides volume default_mode {"key": "s2", "path": "p2", "mode": "0600"}, "0755", "0600", ), ( # Item inherits class default for volume default_mode {"key": "s3", "path": "p3"}, None, "0644", ), ], ) def test_item_mode_propagation_and_override(self, item_config, default_mode, expected): input_config = { "secret_volume_name": "test_vol", "secret_name": item_config["key"], "container_path": "/secrets", "items": [item_config], } if default_mode is not None: input_config["default_mode"] = default_mode context = config_utils.NullConfigContext validated_volume_obj = config_parse.valid_secret_volume(input_config, context) assert validated_volume_obj.items is not None assert len(validated_volume_obj.items) == 1 actual_item_mode = validated_volume_obj.items[0].mode assert actual_item_mode == expected def test_volume_when_items_key_is_omitted(self): input_config = { "secret_volume_name": "vol_no_items", "secret_name": "s_no_items", "container_path": "/secrets", "default_mode": "0400", } context = config_utils.NullConfigContext validated_volume_obj = config_parse.valid_secret_volume(input_config, context) assert validated_volume_obj.default_mode == "0400" assert validated_volume_obj.items is None def test_volume_when_items_is_empty(self): input_config = { "secret_volume_name": "vol_empty_items", "secret_name": "s_empty_items", "container_path": "/secrets", "default_mode": "0700", "items": [], } context = config_utils.NullConfigContext validated_volume_obj = config_parse.valid_secret_volume(input_config, context) assert validated_volume_obj.default_mode == "0700" assert validated_volume_obj.items is not None assert isinstance(validated_volume_obj.items, tuple) assert len(validated_volume_obj.items) == 0 class TestValidSecretVolume: @pytest.mark.parametrize( "config", [ dict( secret_volume_name="abc", secret_name="secret1", container_path="/b/c", default_mode="0644", items=[ dict(key="secret1", path="abcd", mode="7778"), ], ), dict( secret_volume_name="abc", container_path="/b/c", default_mode="0644", items=[ dict(key="secret1", path="abcd", mode="7777"), ], ), dict( secret_volume_name="abc", secret_name="secret1", container_path=123, ), dict( secret_volume_name="abc", secret_name="secret1", container_path="/b/c", items=[dict(key="secret1", path="abcd", mode="7777"), dict(key="secret1", path="abcde", mode="7777")], ), ], ) def test_invalid(self, config): with pytest.raises(ConfigError): config_parse.valid_secret_volume(config, NullConfigContext) def test_wrong_item_key(self): config = dict( secret_volume_name="abc", secret_name="secret1", container_path="/b/c", items=[ dict(key="secret2", path="abc"), ], ) with pytest.raises(ConfigError): config_parse.valid_secret_volume(config, NullConfigContext) @pytest.mark.parametrize( "config", [ dict( secret_volume_name="abc", secret_name="secret1", container_path="/b/c", default_mode="0644", items=[ dict(key="secret1", path="abc"), ], ), dict( secret_volume_name="abc", secret_name="secret1", container_path="/b/c", items=[], ), dict( secret_volume_name="abc", secret_name="secret1", container_path="/b/c", ), ], ) def test_valid(self, config): config_parse.valid_secret_volume(config, NullConfigContext) class TestValidMasterAddress: @pytest.fixture def context(self): return config_utils.NullConfigContext @pytest.mark.parametrize( "url", [ "http://blah.com", "http://blah.com/", "blah.com", "blah.com/", ], ) def test_valid(self, url, context): normalized = "http://blah.com" result = config_parse.valid_master_address(url, context) assert result == normalized @pytest.mark.parametrize( "url", [ "https://blah.com", "http://blah.com/something", "blah.com/other", "http://", "blah.com?a=1", ], ) def test_invalid(self, url, context): with pytest.raises(ConfigError): config_parse.valid_master_address(url, context) class TestValidKubeconfigPaths: @setup def setup_context(self): self.context = config_utils.NullConfigContext @pytest.mark.parametrize( "kubeconfig_path,watcher_kubeconfig_paths", [("/some/kubeconfig.conf", []), ("/another/kube/config", ["a_watcher_kubeconfig"])], ) def test_valid(self, kubeconfig_path, watcher_kubeconfig_paths): k8s_options = { "enabled": True, "kubeconfig_path": kubeconfig_path, "watcher_kubeconfig_paths": watcher_kubeconfig_paths, } assert config_parse.valid_kubernetes_options.validate(k8s_options, self.context) @pytest.mark.parametrize( "kubeconfig_path,watcher_kubeconfig_paths", [ (["/a/kubeconfig/in/a/list"], ["/a/valid/kubeconfig"]), (None, []), ("/some/kubeconfig.conf", "/not/a/list/kubeconfig"), ], ) def test_invalid(self, kubeconfig_path, watcher_kubeconfig_paths): k8s_options = { "enabled": True, "kubeconfig_path": kubeconfig_path, "watcher_kubeconfig_paths": watcher_kubeconfig_paths, } with pytest.raises(ConfigError): config_parse.valid_kubernetes_options.validate(k8s_options, self.context) def test_nonretry(self): k8s_options = { "enabled": True, "kubeconfig_path": "/some/valid/path", "watcher_kubeconfig_paths": [], "non_retryable_exit_codes": 1, } with pytest.raises(ConfigError): config_parse.valid_kubernetes_options.validate(k8s_options, self.context) k8s_options["non_retryable_exit_codes"] = [-12, 1] assert config_parse.valid_kubernetes_options.validate(k8s_options, self.context) class TestValidateStatePersistenceDefaults(TestCase): def test_post_validation_sees_defaults_for_omitted_keys(self): input_config = { "store_type": "dynamodb", "name": "test_state", "table_name": "test_table", "dynamodb_region": "us-west-2", "buffer_size": 5, # max_transact_write_items } original_post_validation = config_parse.ValidateStatePersistence.post_validation post_validation_args = {} def mock_post_validation_side_effect(self_validator, output_dict, config_context): post_validation_args["max_transact_write_items"] = output_dict.get("max_transact_write_items") post_validation_args["buffer_size"] = output_dict.get("buffer_size") return original_post_validation(self_validator, output_dict, config_context) with mock.patch.object( config_parse.ValidateStatePersistence, "post_validation", side_effect=mock_post_validation_side_effect, autospec=True, ) as mock_method: validator = config_parse.ValidateStatePersistence() context = config_utils.NullConfigContext validated_config = validator(input_config, context) mock_method.assert_called_once() assert post_validation_args.get("max_transact_write_items") == 8 assert post_validation_args.get("buffer_size") == 5 assert validated_config.store_type == "dynamodb" assert validated_config.name == "test_state" assert validated_config.table_name == "test_table" assert validated_config.dynamodb_region == "us-west-2" assert validated_config.buffer_size == 5 assert validated_config.max_transact_write_items == 8 def test_post_validation_sees_provided_values(self): input_config = { "store_type": "dynamodb", "name": "test_state", "table_name": "test_table", "dynamodb_region": "us-west-2", "buffer_size": 5, "max_transact_write_items": 25, } original_post_validation = config_parse.ValidateStatePersistence.post_validation post_validation_args = {} def mock_post_validation_side_effect(self_validator, output_dict, config_context): post_validation_args["max_transact_write_items"] = output_dict.get("max_transact_write_items") return original_post_validation(self_validator, output_dict, config_context) with mock.patch.object( config_parse.ValidateStatePersistence, "post_validation", side_effect=mock_post_validation_side_effect, autospec=True, ) as mock_method: validator = config_parse.ValidateStatePersistence() context = config_utils.NullConfigContext validated_config = validator(input_config, context) mock_method.assert_called_once() assert post_validation_args.get("max_transact_write_items") == 25 assert validated_config.max_transact_write_items == 25 assert validated_config.store_type == "dynamodb" assert validated_config.name == "test_state" assert validated_config.table_name == "test_table" assert validated_config.dynamodb_region == "us-west-2" assert validated_config.buffer_size == 5 if __name__ == "__main__": run() ================================================ FILE: tests/config/config_utils_test.py ================================================ import datetime from unittest import mock from testifycompat import assert_equal from testifycompat import assert_in from testifycompat import run from testifycompat import setup from testifycompat import TestCase from tests.assertions import assert_raises from tron.config import config_utils from tron.config import ConfigError from tron.config import schema from tron.config.config_utils import build_list_of_type_validator from tron.config.config_utils import ConfigContext from tron.config.config_utils import valid_identifier class TestUniqueNameDict(TestCase): @setup def setup_dict(self): self.msg = "The key %s was there." self.dict = config_utils.UniqueNameDict(self.msg) def test_set_item_no_conflict(self): self.dict["a"] = "something" assert_in("a", self.dict) def test_set_item_conflict(self): self.dict["a"] = "something" assert_raises(ConfigError, self.dict.__setitem__, "a", "next_thing") class TestValidatorIdentifier(TestCase): def test_valid_identifier_too_long(self): assert_raises(ConfigError, valid_identifier, "a" * 256, mock.Mock()) def test_valid_identifier(self): name = "avalidname" assert_equal(name, valid_identifier(name, mock.Mock())) def test_valid_identifier_invalid_character(self): for name in ["invalid space", "*name", "1numberstarted", 123, ""]: assert_raises(ConfigError, valid_identifier, name, mock.Mock()) class TestBuildListOfTypeValidator(TestCase): @setup def setup_validator(self): self.item_validator = mock.Mock() self.validator = build_list_of_type_validator(self.item_validator) def test_validator_passes(self): items, context = ["one", "two"], mock.create_autospec(ConfigContext) self.validator(items, context) expected = [mock.call(item, context) for item in items] assert_equal(self.item_validator.mock_calls, expected) def test_validator_fails(self): self.item_validator.side_effect = ConfigError items, context = ["one", "two"], mock.create_autospec(ConfigContext) assert_raises(ConfigError, self.validator, items, context) class TestBuildEnumValidator(TestCase): @setup def setup_enum_validator(self): self.enum = dict(a=1, b=2) self.validator = config_utils.build_enum_validator(self.enum) self.context = config_utils.NullConfigContext def test_validate(self): assert_equal(self.validator("a", self.context), "a") assert_equal(self.validator("b", self.context), "b") def test_invalid(self): exception = assert_raises( ConfigError, self.validator, "c", self.context, ) assert_in( "Value at is not in %s: " % str(set(self.enum)), str(exception), ) class TestValidTime(TestCase): @setup def setup_config(self): self.context = config_utils.NullConfigContext def test_valid_time(self): time_spec = config_utils.valid_time("14:32", self.context) assert_equal(time_spec.hour, 14) assert_equal(time_spec.minute, 32) assert_equal(time_spec.second, 0) def test_valid_time_with_seconds(self): time_spec = config_utils.valid_time("14:32:12", self.context) assert_equal(time_spec.hour, 14) assert_equal(time_spec.minute, 32) assert_equal(time_spec.second, 12) def test_valid_time_invalid(self): assert_raises( ConfigError, config_utils.valid_time, "14:32:12:34", self.context, ) assert_raises(ConfigError, config_utils.valid_time, None, self.context) class TestValidTimeDelta(TestCase): @setup def setup_config(self): self.context = config_utils.NullConfigContext def test_valid_time_delta_invalid(self): exception = assert_raises( ConfigError, config_utils.valid_time_delta, "no time", self.context, ) assert_in("not a valid time delta: no time", str(exception)) def test_valid_time_delta_valid_seconds(self): for jitter in [" 82s ", "82 s", "82 sec", "82seconds "]: delta = datetime.timedelta(seconds=82) assert_equal( delta, config_utils.valid_time_delta( jitter, self.context, ), ) def test_valid_time_delta_valid_minutes(self): for jitter in ["10m", "10 m", "10 min", " 10minutes"]: delta = datetime.timedelta(seconds=600) assert_equal( delta, config_utils.valid_time_delta( jitter, self.context, ), ) def test_valid_time_delta_invalid_unit(self): for jitter in ["1 year", "3 mo", "3 months"]: assert_raises( ConfigError, config_utils.valid_time_delta, jitter, self.context, ) class TestConfigContext(TestCase): def test_build_config_context(self): path, nodes, namespace = "path", {1, 2, 3}, "namespace" command_context = mock.MagicMock() parent_context = config_utils.ConfigContext( path, nodes, command_context, namespace, ) child = parent_context.build_child_context("child") assert_equal(child.path, "%s.child" % path) assert_equal(child.nodes, nodes) assert_equal(child.namespace, namespace) assert_equal(child.command_context, command_context) assert not child.partial StubConfigObject = schema.config_object_factory( "StubConfigObject", ["req1", "req2"], ["opt1", "opt2"], ) class StubValidator(config_utils.Validator): config_class = StubConfigObject class TestValidator(TestCase): @setup def setup_validator(self): self.validator = StubValidator() def test_validate_with_none(self): expected_msg = "A StubObject is required" exception = assert_raises( ConfigError, self.validator.validate, None, config_utils.NullConfigContext, ) assert_in(expected_msg, str(exception)) def test_validate_optional_with_none(self): self.validator.optional = True config = self.validator.validate(None, config_utils.NullConfigContext) assert_equal(config, None) if __name__ == "__main__": run() ================================================ FILE: tests/config/manager_test.py ================================================ import os import shutil import tempfile from unittest import mock from testifycompat import assert_equal from testifycompat import run from testifycompat import setup from testifycompat import teardown from testifycompat import TestCase from tests.assertions import assert_raises from tests.testingutils import autospec_method from tron import yaml from tron.config import ConfigError from tron.config import manager from tron.config import schema class TestFromString(TestCase): def test_from_string_valid(self): content = "{'one': 'thing', 'another': 'thing'}\n" actual = manager.from_string(content) expected = {"one": "thing", "another": "thing"} assert_equal(actual, expected) def test_from_string_invalid(self): content = "{} asdf" assert_raises(ConfigError, manager.from_string, content) class TestReadWrite(TestCase): @setup def setup_tempfile(self): self.filename = tempfile.NamedTemporaryFile().name @teardown def teardown_tempfile(self): os.unlink(self.filename) def test_read_write(self): content = {"one": "stars", "two": "beers"} manager.write(self.filename, content) actual = manager.read(self.filename) assert_equal(content, actual) def test_read_raw_write_raw(self): content = "Some string" manager.write_raw(self.filename, content) actual = manager.read_raw(self.filename) assert_equal(content, actual) class TestManifestFile(TestCase): @setup def setup_manifest(self): self.temp_dir = tempfile.mkdtemp() self.manifest = manager.ManifestFile(self.temp_dir) self.manifest.create() @teardown def teardown_dir(self): shutil.rmtree(self.temp_dir) @mock.patch("tron.config.manager.os.path", autospec=True) @mock.patch("tron.config.manager.write", autospec=True) def test_create_exists(self, mock_write, mock_os): mock_os.isfile.return_value = True self.manifest.create() assert not mock_write.call_count def test_create(self): assert_equal(manager.read(self.manifest.filename), {}) def test_add(self): self.manifest.add("zing", "zing.yaml") expected = {"zing": "zing.yaml"} assert_equal(manager.read(self.manifest.filename), expected) def test_delete(self): current = { "one": "a.yaml", "two": "b.yaml", } manager.write(self.manifest.filename, current) self.manifest.delete("one") expected = {"two": "b.yaml"} assert_equal(manager.read(self.manifest.filename), expected) def test_get_file_mapping(self): file_mapping = { "one": "a.yaml", "two": "b.yaml", } manager.write(self.manifest.filename, file_mapping) assert_equal(self.manifest.get_file_mapping(), file_mapping) class TestConfigManager(TestCase): content = {"one": "stars", "two": "other"} raw_content = "{'one': 'stars', 'two': 'other'}\n" @setup def setup_config_manager(self): self.temp_dir = tempfile.mkdtemp() self.manager = manager.ConfigManager(self.temp_dir) self.manifest = mock.create_autospec(manager.ManifestFile) self.manager.manifest = self.manifest @teardown def teardown_dir(self): shutil.rmtree(self.temp_dir) def test_build_file_path(self): path = self.manager.build_file_path("what") assert_equal(path, os.path.join(self.temp_dir, "what.yaml")) def test_build_file_path_with_invalid_chars(self): path = self.manager.build_file_path("/etc/passwd") assert_equal(path, os.path.join(self.temp_dir, "_etc_passwd.yaml")) path = self.manager.build_file_path("../../etc/passwd") assert_equal( path, os.path.join( self.temp_dir, "______etc_passwd.yaml", ), ) def test_read_raw_config(self): name = "name" path = os.path.join(self.temp_dir, name) manager.write(path, self.content) self.manifest.get_file_name.return_value = path config = self.manager.read_raw_config(name) assert_equal(config, yaml.dump(self.content)) def test_write_config(self): name = "filename" path = self.manager.build_file_path(name) self.manifest.get_file_name.return_value = path autospec_method(self.manager.validate_with_fragment) self.manager.write_config(name, self.raw_content) assert_equal(manager.read(path), self.content) self.manifest.get_file_name.assert_called_with(name) assert not self.manifest.add.call_count self.manager.validate_with_fragment.assert_called_with( name, self.content, should_validate_missing_dependency=False, ) def test_write_config_new_name(self): name = "filename2" path = self.manager.build_file_path(name) self.manifest.get_file_name.return_value = None autospec_method(self.manager.validate_with_fragment) self.manager.write_config(name, self.raw_content) assert_equal(manager.read(path), self.content) self.manifest.get_file_name.assert_called_with(name) self.manifest.add.assert_called_with(name, path) @mock.patch("os.remove", autospec=True) def test_delete_config(self, mock_remove): name = "namespace" path = "namespace.yaml" self.manifest.get_file_name.return_value = path self.manager.delete_config(name) self.manifest.delete.assert_called_with(name) mock_remove.assert_called_with(path) @mock.patch("os.remove", autospec=True) def test_delete_missing_namespace(self, mock_remove): name = "namespace" self.manifest.get_file_name.return_value = None self.manager.delete_config(name) assert_equal(mock_remove.call_count, 0) @mock.patch( "tron.config.manager.JobGraph", autospec=True, ) @mock.patch( "tron.config.manager.config_parse.ConfigContainer", autospec=True, ) def test_validate_with_fragment(self, mock_config_container, mock_job_graph): name = "the_name" name_mapping = {"something": "content", name: "old_content"} autospec_method(self.manager.get_config_name_mapping) self.manager.get_config_name_mapping.return_value = name_mapping self.manager.validate_with_fragment(name, self.content) expected_mapping = dict(name_mapping) expected_mapping[name] = self.content mock_config_container.create.assert_called_with(expected_mapping) mock_job_graph.assert_called_once_with( mock_config_container.create.return_value, should_validate_missing_dependency=True, ) @mock.patch("tron.config.manager.read", autospec=True) @mock.patch( "tron.config.manager.config_parse.ConfigContainer", autospec=True, ) def test_load(self, mock_config_container, mock_read): content_items = self.content.items() self.manifest.get_file_mapping().return_value = content_items container = self.manager.load() self.manifest.get_file_mapping.assert_called_with() assert_equal(container, mock_config_container.create.return_value) expected = {name: call.return_value for ((name, _), call) in zip(content_items, mock_read.mock_calls)} mock_config_container.create.assert_called_with(expected) def test_get_hash_default(self): self.manifest.__contains__.return_value = False hash_digest = self.manager.get_hash("name") assert_equal(hash_digest, self.manager.DEFAULT_HASH) def test_get_hash(self): content = "OkOkOk" autospec_method(self.manager.read_raw_config, return_value=content) self.manifest.__contains__.return_value = True hash_digest = self.manager.get_hash("name") assert_equal(hash_digest, manager.hash_digest(content)) class TestCreateNewConfig(TestCase): @mock.patch("tron.config.manager.os.makedirs", autospec=True) @mock.patch("tron.config.manager.ManifestFile", autospec=True) @mock.patch("tron.config.manager.write_raw", autospec=True) def test_create_new_config(self, mock_write, mock_manifest, mock_makedirs): path, master_content = "/bogus/path/", mock.Mock() filename = "/bogus/path/MASTER.yaml" manifest = mock_manifest.return_value manifest.get_file_name.return_value = None manager.create_new_config(path, master_content) mock_makedirs.assert_called_with(path) mock_write.assert_called_with(filename, master_content) manifest.create.assert_called_with() manifest.add.assert_called_with(schema.MASTER_NAMESPACE, filename) if __name__ == "__main__": run() ================================================ FILE: tests/config/schedule_parse_test.py ================================================ import datetime from unittest import mock from testifycompat import assert_equal from testifycompat import assert_raises from testifycompat import run from testifycompat import TestCase from tron.config import config_utils from tron.config import ConfigError from tron.config import schedule_parse class TestPadSequence(TestCase): def test_pad_sequence_short(self): expected = [0, 1, 2, 3, None, None] assert_equal(schedule_parse.pad_sequence(range(4), 6), expected) def test_pad_sequence_long(self): expected = [0, 1, 2, 3] assert_equal(schedule_parse.pad_sequence(range(6), 4), expected) def test_pad_sequence_exact(self): expected = [0, 1, 2, 3] assert_equal(schedule_parse.pad_sequence(range(4), 4), expected) def test_pad_sequence_empty(self): expected = ["a", "a"] assert_equal(schedule_parse.pad_sequence([], 2, "a"), expected) def test_pad_negative_size(self): assert_equal(schedule_parse.pad_sequence([], -2, "a"), []) class TestScheduleConfigFromString(TestCase): @mock.patch( "tron.config.schedule_parse.parse_groc_expression", autospec=True, ) def test_groc_config(self, mock_parse_groc): schedule = "every Mon,Wed at 12:00" context = config_utils.NullConfigContext config = schedule_parse.schedule_config_from_string(schedule, context) assert_equal(config, mock_parse_groc.return_value) generic_config = schedule_parse.ConfigGenericSchedule( "groc daily", schedule, None, ) mock_parse_groc.assert_called_with(generic_config, context) class TestValidScheduler(TestCase): @mock.patch("tron.config.schedule_parse.schedulers", autospec=True) def assert_validation(self, schedule, expected, mock_schedulers): context = config_utils.NullConfigContext config = schedule_parse.valid_schedule(schedule, context) mock_schedulers.__getitem__.assert_called_with("cron") func = mock_schedulers.__getitem__.return_value assert_equal(config, func.return_value) func.assert_called_with(expected, context) def test_cron_from_dict(self): schedule = {"type": "cron", "value": "* * * * *"} config = schedule_parse.ConfigGenericSchedule( "cron", schedule["value"], datetime.timedelta(), ) self.assert_validation(schedule, config) def test_cron_from_dict_with_jitter(self): schedule = {"type": "cron", "value": "* * * * *", "jitter": "5 min"} config = schedule_parse.ConfigGenericSchedule( "cron", schedule["value"], datetime.timedelta(minutes=5), ) self.assert_validation(schedule, config) class TestValidCronScheduler(TestCase): _suites = ["integration"] def validate(self, line): config = schedule_parse.ConfigGenericSchedule("cron", line, None) context = config_utils.NullConfigContext return schedule_parse.valid_cron_scheduler(config, context) def test_valid_config(self): config = self.validate("5 0 L * *") assert_equal(config.minutes, [5]) assert_equal(config.months, None) assert_equal(config.monthdays, ["LAST"]) def test_invalid_config(self): assert_raises(ConfigError, self.validate, "* * *") class TestValidDailyScheduler(TestCase): def validate(self, config): context = config_utils.NullConfigContext config = schedule_parse.ConfigGenericSchedule("daily", config, None) return schedule_parse.valid_daily_scheduler(config, context) def assert_parse(self, config, expected): config = self.validate(config) expected = schedule_parse.ConfigDailyScheduler(*expected, jitter=None) assert_equal(config, expected) def test_valid_daily_scheduler_start_time(self): expected = ("14:32 ", 14, 32, 0, set()) self.assert_parse("14:32", expected) def test_valid_daily_scheduler_just_days(self): expected = ("00:00:00 MWS", 0, 0, 0, {1, 3, 6}) self.assert_parse("00:00:00 MWS", expected) def test_valid_daily_scheduler_time_and_day(self): expected = ("17:02:44 SU", 17, 2, 44, {0, 6}) self.assert_parse("17:02:44 SU", expected) def test_valid_daily_scheduler_invalid_start_time(self): assert_raises(ConfigError, self.validate, "5 MWF") assert_raises(ConfigError, self.validate, "05:30:45:45 MWF") assert_raises(ConfigError, self.validate, "25:30:45 MWF") def test_valid_daily_scheduler_invalid_days(self): assert_raises(ConfigError, self.validate, "SUG") assert_raises(ConfigError, self.validate, "3") if __name__ == "__main__": run() ================================================ FILE: tests/core/__init__.py ================================================ ================================================ FILE: tests/core/action_test.py ================================================ import pytest from tron.config.schema import ConfigAction from tron.config.schema import ConfigConstraint from tron.config.schema import ConfigFieldSelectorSource from tron.config.schema import ConfigNodeAffinity from tron.config.schema import ConfigParameter from tron.config.schema import ConfigProjectedSAVolume from tron.config.schema import ConfigSecretSource from tron.config.schema import ConfigSecretVolume from tron.config.schema import ConfigSecretVolumeItem from tron.config.schema import ConfigTopologySpreadConstraints from tron.config.schema import ConfigVolume from tron.core.action import Action from tron.core.action import ActionCommandConfig class TestAction: @pytest.mark.parametrize("disk", [600.0, None]) def test_from_config_full(self, disk): config = ConfigAction( name="ted", command="do something", node="first", executor="ssh", cpus=1, mem=100, disk=disk, # default: 1024.0 constraints=[ ConfigConstraint( attribute="pool", operator="LIKE", value="default", ), ], docker_image="fake-docker.com:400/image", docker_parameters=[ ConfigParameter( key="test", value=123, ), ], env={"TESTING": "true"}, secret_env={"TEST_SECRET": ConfigSecretSource(secret_name="tron-secret-svc-sec--A", key="sec_A")}, secret_volumes=[ ConfigSecretVolume( secret_volume_name="secretvolumename", secret_name="secret", container_path="/b", default_mode="0644", items=[ConfigSecretVolumeItem(key="key", path="path", mode="0755")], ), ], extra_volumes=[ ConfigVolume( host_path="/tmp", container_path="/nail/tmp", mode="RO", ), ], trigger_downstreams=True, triggered_by=["foo.bar"], ) new_action = Action.from_config(config) assert new_action.name == config.name assert new_action.node_pool is None assert new_action.executor == config.executor assert new_action.trigger_downstreams is True assert new_action.triggered_by == ["foo.bar"] command_config = new_action.command_config assert command_config.command == config.command assert command_config.cpus == config.cpus assert command_config.mem == config.mem assert command_config.disk == (600.0 if disk else 1024.0) assert command_config.constraints == {("pool", "LIKE", "default")} assert command_config.docker_image == config.docker_image assert command_config.docker_parameters == {("test", 123)} assert command_config.env == config.env assert command_config.secret_env == config.secret_env # cant do direct tuple equality, since this is not hashable assert command_config.secret_volumes == config.secret_volumes assert command_config.extra_volumes == {("/nail/tmp", "/tmp", "RO")} def test_from_config_none_values(self): config = ConfigAction( name="ted", command="do something", node="first", executor="ssh", ) new_action = Action.from_config(config) assert new_action.name == config.name assert new_action.executor == config.executor command_config = new_action.command_config assert command_config.command == config.command assert command_config.constraints == set() assert command_config.docker_image is None assert command_config.docker_parameters == set() assert command_config.env == {} assert command_config.secret_env == {} assert command_config.secret_volumes == [] assert command_config.extra_volumes == set() @pytest.fixture def action_command_config_json(self): raw_json = """ { "command": "echo 'Hello, World!'", "cpus": 1.0, "mem": 512.0, "disk": 1024.0, "cap_add": ["NET_ADMIN"], "cap_drop": ["MKNOD"], "constraints": [ { "attribute": "pool", "operator": "LIKE", "value": "default" } ], "docker_image": "fake-docker.com:400/image", "docker_parameters": [ { "key": "test", "value": 123 } ], "env": {"TESTING": "true"}, "secret_env": { "TEST_SECRET": { "secret_name": "tron-secret-svc-sec--A", "key": "sec_A" } }, "secret_volumes": [ { "secret_volume_name": "secretvolumename", "secret_name": "secret", "container_path": "/b", "default_mode": "0644", "items": [ { "key": "key", "path": "path", "mode": "0755" } ] } ], "projected_sa_volumes": [ { "container_path": "/var/run/secrets/whatever", "audience": "for.bar.com", "expiration_seconds": 3600 } ], "extra_volumes": [ { "container_path": "/tmp", "host_path": "/home/tmp", "mode": "RO" } ], "node_affinities": [ { "key": "topology.kubernetes.io/zone", "operator": "In", "value": ["us-west-1a", "us-west-1c"] } ], "topology_spread_constraints": [ { "topology_key": "zone", "max_skew": 1, "when_unsatisfiable": "DoNotSchedule", "label_selector": { "match_labels": { "app": "myapp" } } } ], "labels": {"app": "myapp"}, "idempotent": true, "annotations": {"annotation_key": "annotation_value"}, "service_account_name": "default", "ports": [8080, 9090], "field_selector_env": { "key": { "field_path": "value" } }, "node_selectors": {"key": "node-A"} } """ return raw_json def test_action_command_config_from_json(self, action_command_config_json): result = ActionCommandConfig.from_json(action_command_config_json) expected = { "command": "echo 'Hello, World!'", "cpus": 1.0, "mem": 512.0, "disk": 1024.0, "cap_add": ["NET_ADMIN"], "cap_drop": ["MKNOD"], "constraints": [ConfigConstraint(attribute="pool", operator="LIKE", value="default")], "docker_image": "fake-docker.com:400/image", "docker_parameters": [ConfigParameter(key="test", value=123)], "env": {"TESTING": "true"}, "secret_env": {"TEST_SECRET": ConfigSecretSource(secret_name="tron-secret-svc-sec--A", key="sec_A")}, "secret_volumes": [ ConfigSecretVolume( secret_volume_name="secretvolumename", secret_name="secret", container_path="/b", default_mode="0644", items=[{"key": "key", "path": "path", "mode": "0755"}], ) ], "projected_sa_volumes": [ ConfigProjectedSAVolume( container_path="/var/run/secrets/whatever", audience="for.bar.com", expiration_seconds=3600, ) ], "extra_volumes": [ConfigVolume(container_path="/tmp", host_path="/home/tmp", mode="RO")], "node_affinities": [ ConfigNodeAffinity(key="topology.kubernetes.io/zone", operator="In", value=["us-west-1a", "us-west-1c"]) ], "topology_spread_constraints": [ ConfigTopologySpreadConstraints( topology_key="zone", max_skew=1, when_unsatisfiable="DoNotSchedule", label_selector={"match_labels": {"app": "myapp"}}, ) ], "labels": {"app": "myapp"}, "annotations": {"annotation_key": "annotation_value"}, "service_account_name": "default", "ports": [8080, 9090], "idempotent": True, "node_selectors": {"key": "node-A"}, "field_selector_env": {"key": ConfigFieldSelectorSource(field_path="value")}, } assert result == expected ================================================ FILE: tests/core/actiongraph_test.py ================================================ from unittest import mock from testifycompat import assert_equal from testifycompat import assert_raises from testifycompat import run from testifycompat import setup from testifycompat import TestCase from tron.core import actiongraph class TestActionGraph(TestCase): @setup def setup_graph(self): self.action_names = [ "base_one", "base_two", "dep_one", "dep_one_one", "dep_multi", ] self.action_map = {} for name in self.action_names: self.action_map[name] = mock.MagicMock() self.action_map[name].name = name self.required_actions = { "base_one": set(), "base_two": set(), "dep_multi": {"dep_one_one", "base_two"}, "dep_one_one": {"dep_one"}, "dep_one": {"base_one"}, } self.required_triggers = { "base_one": {"MASTER.otherjob.first"}, "base_two": set(), "dep_multi": set(), "dep_one_one": set(), "dep_one": set(), } self.action_graph = actiongraph.ActionGraph(self.action_map, self.required_actions, self.required_triggers) def test_get_dependencies(self): assert self.action_graph.get_dependencies("not_in_job") == [] assert self.action_graph.get_dependencies("base_one") == [] assert self.action_graph.get_dependencies("base_one", include_triggers=True)[0].name == "MASTER.otherjob.first" assert sorted(d.name for d in self.action_graph.get_dependencies("dep_multi")) == sorted( [ "dep_one_one", "base_two", ] ) def test_names(self): assert sorted(self.action_graph.names()) == sorted(self.action_names) assert sorted(self.action_graph.names(include_triggers=True)) == sorted( self.action_names + ["MASTER.otherjob.first"], ) def test__getitem__(self): assert_equal( self.action_graph["base_one"], self.action_map["base_one"], ) def test__getitem__miss(self): assert_raises(KeyError, lambda: self.action_graph["unknown"]) def test__eq__(self): other_graph = mock.MagicMock( action_map=self.action_map, required_actions=self.required_actions, required_triggers=self.required_triggers, ) assert_equal(self.action_graph, other_graph) other_graph.required_actions = None assert not self.action_graph == other_graph def test__ne__(self): other_graph = mock.MagicMock() assert self.action_graph != other_graph if __name__ == "__main__": run() ================================================ FILE: tests/core/actionrun_test.py ================================================ import datetime import shutil import tempfile from unittest import mock from unittest.mock import MagicMock import pytest from tests.assertions import assert_length from tests.testingutils import autospec_method from tron import actioncommand from tron import node from tron.actioncommand import SubprocessActionRunnerFactory from tron.config.schema import ConfigConstraint from tron.config.schema import ConfigParameter from tron.config.schema import ConfigVolume from tron.config.schema import ExecutorTypes from tron.core import actiongraph from tron.core import jobrun from tron.core.action import ActionCommandConfig from tron.core.actionrun import ActionCommand from tron.core.actionrun import ActionRun from tron.core.actionrun import ActionRunAttempt from tron.core.actionrun import ActionRunCollection from tron.core.actionrun import ActionRunFactory from tron.core.actionrun import eager_all from tron.core.actionrun import INITIAL_RECOVER_DELAY from tron.core.actionrun import KubernetesActionRun from tron.core.actionrun import MAX_RECOVER_TRIES from tron.core.actionrun import MesosActionRun from tron.core.actionrun import min_filter from tron.core.actionrun import SSHActionRun from tron.serialize import filehandler @pytest.fixture def output_path(): output_path = filehandler.OutputPath(tempfile.mkdtemp()) yield output_path shutil.rmtree(output_path.base, ignore_errors=True) @pytest.fixture def mock_current_time(): with mock.patch( "tron.core.actionrun.timeutils.current_time", autospec=True, ) as mock_current_time: yield mock_current_time class TestMinFilter: def test_min_filter(self): seq = [None, 2, None, 7, None, 9, 10, 12, 1] assert min_filter(seq) == 1 class TestEagerAll: def test_all_true(self): assert eager_all(range(1, 5)) def test_all_false(self): assert not eager_all(0 for _ in range(7)) def test_full_iteration(self): seq = iter([1, 0, 3, 0, 5]) assert not eager_all(seq) with pytest.raises(StopIteration): next(seq) class TestActionRunFactory: @pytest.fixture(autouse=True) def setup_action_runs(self): self.run_time = datetime.datetime(2012, 3, 14, 15, 9, 26) a1 = MagicMock() a1.name = "act1" a1.command_config = ActionCommandConfig(command="do action1") a2 = MagicMock() a2.name = "act2" actions = [a1, a2] self.action_graph = actiongraph.ActionGraph( {a.name: a for a in actions}, {"act1": set(), "act2": set()}, {"act1": set(), "act2": set()}, ) mock_node = mock.create_autospec(node.Node) self.job_run = jobrun.JobRun( "jobname", 7, self.run_time, mock_node, action_graph=self.action_graph, ) self.action_runner = mock.create_autospec( actioncommand.SubprocessActionRunnerFactory, ) @pytest.fixture def state_data(self): command_config = self.action_graph.action_map["act1"].command_config.state_data # State data with command config and retries. yield { "job_run_id": "job_run_id", "action_name": "act1", "state": "succeeded", "run_time": "the_run_time", "start_time": None, "end_time": None, "attempts": [dict(command_config=command_config, start_time="start")], "node_name": "anode", } def test_build_action_run_collection(self): collection = ActionRunFactory.build_action_run_collection( self.job_run, self.action_runner, ) assert collection.action_graph == self.action_graph assert "act1" in collection.run_map assert "act2" in collection.run_map assert len(collection.run_map) == 2 assert collection.run_map["act1"].action_name == "act1" def test_action_run_collection_from_state(self, state_data): state_data = [state_data] cleanup_command_config = dict(command="do action1") cleanup_action_state_data = { "job_run_id": "job_run_id", "action_name": "cleanup", "state": "succeeded", "run_time": self.run_time, "start_time": None, "end_time": None, "attempts": [ dict( command_config=cleanup_command_config, rendered_command="do action1", start_time="start", end_time="end", exit_status=0, ), ], "node_name": "anode", "action_runner": { "status_path": "/tmp/foo", "exec_path": "/bin/foo", }, } collection = ActionRunFactory.action_run_collection_from_state( self.job_run, state_data, cleanup_action_state_data, ) assert collection.action_graph == self.action_graph assert_length(collection.run_map, 2) assert collection.run_map["act1"].action_name == "act1" assert collection.run_map["cleanup"].action_name == "cleanup" def test_build_run_for_action(self): expected_command = "doit" action = MagicMock( node_pool=None, is_cleanup=False, command_config=ActionCommandConfig(command=expected_command), ) action.name = "theaction" action_run = ActionRunFactory.build_run_for_action( self.job_run, action, self.action_runner, ) assert action_run.job_run_id == self.job_run.id assert action_run.node == self.job_run.node assert action_run.action_name == action.name assert not action_run.is_cleanup assert action_run.command == expected_command def test_build_run_for_action_with_node(self): expected_command = "doit" action = MagicMock( node_pool=None, is_cleanup=True, command_config=ActionCommandConfig(command=expected_command), ) action.node_pool = mock.create_autospec(node.NodePool) action_run = ActionRunFactory.build_run_for_action( self.job_run, action, self.action_runner, ) assert action_run.job_run_id == self.job_run.id assert action_run.node == action.node_pool.next() assert action_run.is_cleanup assert action_run.action_name == action.name assert action_run.command == expected_command def test_build_run_for_ssh_action(self): action = MagicMock( name="theaction", command="doit", executor=ExecutorTypes.ssh.value, ) action_run = ActionRunFactory.build_run_for_action( self.job_run, action, self.action_runner, ) assert action_run.__class__ == SSHActionRun def test_build_run_for_mesos_action(self): command_config = MagicMock( cpus=10, mem=500, disk=600, constraints=[["pool", "LIKE", "default"]], docker_image="fake-docker.com:400/image", docker_parameters=[ { "key": "test", "value": 123, } ], env={"TESTING": "true"}, extra_volumes=[ { "path": "/tmp", } ], ) action = MagicMock( name="theaction", command="doit", executor=ExecutorTypes.mesos.value, command_config=command_config, ) action_run = ActionRunFactory.build_run_for_action( self.job_run, action, self.action_runner, ) assert action_run.__class__ == MesosActionRun assert action_run.command_config.cpus == command_config.cpus assert action_run.command_config.mem == command_config.mem assert action_run.command_config.disk == command_config.disk assert action_run.command_config.constraints == command_config.constraints assert action_run.command_config.docker_image == command_config.docker_image assert action_run.command_config.docker_parameters == command_config.docker_parameters assert action_run.command_config.env == command_config.env assert action_run.command_config.extra_volumes == command_config.extra_volumes def test_action_run_from_state_ssh(self, state_data): action_run = ActionRunFactory.action_run_from_state( self.job_run, state_data, ) assert action_run.job_run_id == state_data["job_run_id"] assert not action_run.is_cleanup assert action_run.__class__ == SSHActionRun def test_action_run_from_state_mesos(self, state_data): state_data["executor"] = ExecutorTypes.mesos.value action_run = ActionRunFactory.action_run_from_state( self.job_run, state_data, ) assert action_run.job_run_id == state_data["job_run_id"] action_name = state_data["action_name"] assert action_run.command_config == self.action_graph.action_map[action_name].command_config assert not action_run.is_cleanup assert action_run.__class__ == MesosActionRun def test_action_run_from_state_kubernetes(self, state_data): state_data["executor"] = ExecutorTypes.kubernetes.value action_run = ActionRunFactory.action_run_from_state( self.job_run, state_data, ) assert action_run.job_run_id == state_data["job_run_id"] action_name = state_data["action_name"] assert action_run.command_config == self.action_graph.action_map[action_name].command_config assert not action_run.is_cleanup assert action_run.__class__ == KubernetesActionRun def test_action_run_from_state_spark(self, state_data): state_data["executor"] = ExecutorTypes.spark.value action_run = ActionRunFactory.action_run_from_state( self.job_run, state_data, ) assert action_run.job_run_id == state_data["job_run_id"] action_name = state_data["action_name"] assert action_run.command_config == self.action_graph.action_map[action_name].command_config assert not action_run.is_cleanup assert action_run.__class__ == KubernetesActionRun class TestActionRun: @pytest.fixture(autouse=True) def setup_action_run(self, output_path): self.action_runner = actioncommand.NoActionRunnerFactory() self.command = "do command {actionname}" self.rendered_command = "do command action_name" self.action_run = ActionRun( job_run_id="ns.id.0", name="action_name", node=mock.create_autospec(node.Node), command_config=ActionCommandConfig(command=self.command), output_path=output_path, action_runner=self.action_runner, ) # These should be implemented in subclasses, we don't care here self.action_run.submit_command = mock.Mock() self.action_run.stop = mock.Mock() self.action_run.kill = mock.Mock() def test_init_state(self): assert self.action_run.state == ActionRun.SCHEDULED def test_ready_state(self): self.action_run.ready() assert self.action_run.state == ActionRun.WAITING def test_start(self): self.action_run.machine.transition("ready") assert self.action_run.start() assert self.action_run.submit_command.call_count == 1 assert self.action_run.is_starting assert self.action_run.start_time def test_start_bad_state(self): self.action_run.fail() assert not self.action_run.start() @mock.patch("tron.core.actionrun.log", autospec=True) def test_start_invalid_command(self, _log): self.action_run.original_command = "{notfound}" self.action_run.machine.transition("ready") assert not self.action_run.start() assert self.action_run.is_failed assert self.action_run.exit_status == -1 def test_success(self): assert self.action_run.ready() self.action_run.machine.transition("start") self.action_run.machine.transition("started") assert self.action_run.is_running assert self.action_run.success() assert not self.action_run.is_running assert self.action_run.is_done assert self.action_run.end_time assert self.action_run.exit_status == 0 def test_success_emits_not(self): self.action_run.machine.transition("start") self.action_run.machine.transition("started") self.action_run.trigger_downstreams = None self.action_run.emit_triggers = mock.Mock() assert self.action_run.success() assert self.action_run.emit_triggers.call_count == 0 def test_sucess_emits_not_invalid_transition(self): self.action_run.trigger_downstreams = True self.action_run.machine.check = mock.Mock(return_value=False) self.action_run.emit_triggers = mock.Mock() assert not self.action_run.success() assert self.action_run.emit_triggers.call_count == 0 def test_success_emits_on_true(self): self.action_run.machine.transition("start") self.action_run.machine.transition("started") self.action_run.trigger_downstreams = True self.action_run.emit_triggers = mock.Mock() assert self.action_run.success() assert self.action_run.emit_triggers.call_count == 1 def test_success_emits_on_dict(self): self.action_run.machine.transition("start") self.action_run.machine.transition("started") self.action_run.trigger_downstreams = dict(foo="bar") self.action_run.emit_triggers = mock.Mock() assert self.action_run.success() assert self.action_run.emit_triggers.call_count == 1 @mock.patch("tron.core.actionrun.EventBus", autospec=True) def test_emit_triggers(self, eventbus): self.action_run.context = {"shortdate": "foo"} self.action_run.trigger_downstreams = True self.action_run.emit_triggers() self.action_run.trigger_downstreams = dict(foo="bar") self.action_run.emit_triggers() assert eventbus.publish.mock_calls == [ mock.call("ns.id.action_name.shortdate.foo"), mock.call("ns.id.action_name.foo.bar"), ] def test_failure(self): self.action_run._exit_unsuccessful(1) assert not self.action_run.is_running assert self.action_run.is_done assert self.action_run.end_time assert self.action_run.exit_status == 1 def test_failure_bad_state(self): self.action_run.fail(444) assert not self.action_run.fail(123) assert self.action_run.exit_status == 444 def test_skip(self): assert not self.action_run.is_running self.action_run.ready() assert self.action_run.start() assert self.action_run.fail(-1) assert self.action_run.skip() assert self.action_run.is_skipped def test_skip_bad_state(self): assert not self.action_run.skip() def test_render_command(self): self.action_run.context = {"stars": "bright"} bare_command = "{stars}" assert self.action_run.render_command(bare_command) == "bright" def test_command_not_yet_rendered(self): assert self.action_run.command == self.action_run.command_config.command def test_command_already_rendered(self): last_attempt = self.action_run.create_attempt() assert self.action_run.command == last_attempt.rendered_command @mock.patch("tron.core.actionrun.log", autospec=True) def test_command_failed_render(self, _log): bare_command = "{this_is_missing}" assert self.action_run.render_command(bare_command) == ActionRun.FAILED_RENDER def test_is_complete(self): self.action_run.machine.state = ActionRun.SUCCEEDED assert self.action_run.is_complete self.action_run.machine.state = ActionRun.SKIPPED assert self.action_run.is_complete self.action_run.machine.state = ActionRun.RUNNING assert not self.action_run.is_complete def test_is_broken(self): self.action_run.machine.state = ActionRun.UNKNOWN assert self.action_run.is_broken self.action_run.machine.state = ActionRun.FAILED assert self.action_run.is_broken self.action_run.machine.state = ActionRun.WAITING assert not self.action_run.is_broken def test__getattr__(self): assert not self.action_run.is_succeeded assert not self.action_run.is_failed assert not self.action_run.is_queued assert self.action_run.is_scheduled assert self.action_run.cancel() assert self.action_run.is_cancelled def test__getattr__missing_attribute(self): with pytest.raises(AttributeError): self.action_run.__getattr__("is_not_a_real_state") def test_auto_retry(self, mock_current_time): # One timestamp for start and end of each attempt, plus final end time mock_current_time.side_effect = [1, 2, 3, 4, 5, 6, 7] self.action_run.retries_remaining = 2 self.action_run.create_attempt() self.action_run.machine.transition("start") assert self.action_run._exit_unsuccessful(-1) assert self.action_run.is_starting assert self.action_run.retries_remaining == 1 assert self.action_run._exit_unsuccessful(-1) assert self.action_run.retries_remaining == 0 assert not self.action_run.is_failed assert self.action_run._exit_unsuccessful(-2) assert self.action_run.retries_remaining == 0 assert self.action_run.is_failed assert self.action_run.exit_statuses == [-1, -1, -2] assert len(self.action_run.attempts) == 3 for i, attempt in enumerate(self.action_run.attempts): assert attempt.start_time == i * 2 + 1 assert attempt.end_time == (i + 1) * 2 def test_auto_retry_command_config_change(self, mock_current_time): self.action_run.retries_remaining = 1 self.action_run.create_attempt() self.action_run.machine.transition("start") # If the command_config gets reconfigured later, auto retry # still uses the original command by default. self.action_run.command_config = ActionCommandConfig(command="new") assert self.action_run._exit_unsuccessful(-1) assert self.action_run._exit_unsuccessful(-1) assert len(self.action_run.attempts) == 2 for i, attempt in enumerate(self.action_run.attempts): assert attempt.rendered_command == self.rendered_command def test_no_auto_retry_on_fail_not_running(self): self.action_run.retries_remaining = 2 self.action_run.fail() assert self.action_run.retries_remaining == -1 assert self.action_run.is_failed assert self.action_run.exit_statuses == [] assert self.action_run.exit_status is None def test_no_auto_retry_on_fail_running(self): self.action_run.retries_remaining = 2 self.action_run.create_attempt() self.action_run.machine.transition("start") self.action_run.fail() assert self.action_run.retries_remaining == -1 assert self.action_run.is_failed assert self.action_run.exit_statuses == [None] assert self.action_run.exit_status is None def test_auto_retry_already_done(self): # If someone transitions the action before it # is done to success/fail, the action # should not automatically retry when the command # completes. self.action_run.retries_remaining = 2 self.action_run.create_attempt() self.action_run.machine.transition("start") self.action_run.machine.transition("started") # Action gets manually transitioned to success with tronctl self.action_run.machine.transition("success") assert self.action_run.is_succeeded # Command later fails # Does not start a command assert not self.action_run._exit_unsuccessful(-1) # Still succeeded, not starting assert self.action_run.is_succeeded def test_manual_retry(self, mock_current_time): mock_current_time.side_effect = [1, 2, 3, 4] self.action_run.retries_remaining = None failed_attempt = self.action_run.create_attempt() self.action_run.machine.transition("start") self.action_run.fail(-1) assert failed_attempt.end_time == 2 assert failed_attempt.exit_status == -1 self.action_run.retry() assert self.action_run.is_starting assert self.action_run.exit_statuses == [-1] assert self.action_run.retries_remaining == 0 # Last attempt should be unchanged assert failed_attempt.end_time == 2 assert failed_attempt.exit_status == -1 def test_manual_retry_use_new_command(self, mock_current_time): mock_current_time.side_effect = [1, 2, 3, 4] self.action_run.retries_remaining = None self.action_run.create_attempt() self.action_run.machine.transition("start") self.action_run.fail(-1) # Change the command config self.action_run.command_config = ActionCommandConfig(command="new") self.action_run.retry(original_command=False) assert self.action_run.is_starting assert self.action_run.last_attempt.rendered_command == "new" @mock.patch("twisted.internet.reactor.callLater", autospec=True) def test_retries_delay(self, callLater): self.action_run.retries_delay = datetime.timedelta() self.action_run.retries_remaining = 2 self.action_run.machine.transition("start") callLater.return_value = "delayed call" assert self.action_run._exit_unsuccessful(-1) assert self.action_run.in_delay == "delayed call" class TestActionRunFactoryTriggerTimeout: def test_trigger_timeout_default(self): today = datetime.datetime.today() day = datetime.timedelta(days=1) tomorrow = today + day action_run = ActionRunFactory.build_run_for_action( mock.Mock(run_time=today), mock.Mock(trigger_timeout=None), mock.Mock(), ) assert action_run.trigger_timeout_timestamp == tomorrow.timestamp() def test_trigger_timeout_custom(self): today = datetime.datetime.today() hour = datetime.timedelta(hours=1) target = today + hour action_run = ActionRunFactory.build_run_for_action( mock.Mock(run_time=today), mock.Mock(trigger_timeout=hour), mock.Mock(), ) assert action_run.trigger_timeout_timestamp == target.timestamp() class TestActionRunTriggerTimeout: @pytest.fixture(autouse=True) def setup_teardown(self): self.command = mock.Mock() self.rendered_command = "do command action_name" self.action_run = ActionRun( job_run_id="ns.id.0", name="action_name", command_config=ActionCommandConfig(command=self.command), triggered_by=["hello"], node=mock.Mock(), output_path=mock.Mock(), action_runner=mock.Mock(), trigger_timeout_timestamp=mock.Mock(), ) self.action_run.submit_command = mock.Mock() self.action_run.stop = mock.Mock() self.action_run.kill = mock.Mock() def test_cleanup_clears_trigger_timeout(self): self.action_run.clear_trigger_timeout = MagicMock() self.action_run.cleanup() self.action_run.clear_trigger_timeout.assert_called_with() def test_clear_trigger_timeout(self): timeout_call = MagicMock() self.action_run.trigger_timeout_call = timeout_call self.action_run.clear_trigger_timeout() assert self.action_run.trigger_timeout_call is None timeout_call.cancel.assert_called_with() @mock.patch("tron.core.actionrun.EventBus", autospec=True) @mock.patch("tron.core.actionrun.reactor", autospec=True) def test_setup_subscriptions_no_triggers(self, reactor, eventbus): self.action_run.triggered_by = [] self.action_run.setup_subscriptions() assert not reactor.callLater.called assert not eventbus.subscribe.called @mock.patch("tron.core.actionrun.EventBus", autospec=True) @mock.patch("tron.core.actionrun.reactor", autospec=True) def test_setup_subscriptions_no_remaining(self, reactor, eventbus): self.action_run.triggered_by = ["hello"] self.action_run.trigger_timeout_timestamp = None eventbus.has_event.return_value = True self.action_run.setup_subscriptions() assert not reactor.callLater.called assert not eventbus.subscribe.called assert eventbus.has_event.call_args_list == [mock.call("hello")] @mock.patch("tron.core.actionrun.reactor", autospec=True) def test_setup_subscriptions_timeout_in_future(self, reactor, mock_current_time): now = datetime.datetime.now() mock_current_time.return_value = now self.action_run.trigger_timeout_timestamp = now.timestamp() + 10 self.action_run.setup_subscriptions() reactor.callLater.assert_called_once_with( 10.0, self.action_run.trigger_timeout_reached, ) @mock.patch("tron.core.actionrun.reactor", autospec=True) def test_setup_subscriptions_timeout_in_past(self, reactor, mock_current_time): now = datetime.datetime.now() mock_current_time.return_value = now self.action_run.trigger_timeout_timestamp = now.timestamp() - 10 self.action_run.setup_subscriptions() reactor.callLater.assert_called_once_with( 1, self.action_run.trigger_timeout_reached, ) @mock.patch("tron.core.actionrun.EventBus", autospec=True) def test_trigger_timeout_reached_no_remaining_notifies(self, eventbus): self.action_run.notify = MagicMock() self.action_run.triggered_by = ["hello"] eventbus.has_event.return_value = True self.action_run.trigger_timeout_reached() assert self.action_run.notify.called @mock.patch("tron.core.actionrun.EventBus", autospec=True) def test_trigger_timeout_reached_with_remaining_fails(self, eventbus): self.action_run.fail = MagicMock() self.action_run.triggered_by = ["hello"] eventbus.has_event.return_value = False self.action_run.trigger_timeout_reached() assert self.action_run.fail.called def test_done_clears_trigger_timeout_call(self): self.action_run.machine.check = mock.Mock(return_value=True) self.action_run.transition_and_notify = MagicMock() self.action_run.triggered_by = [] self.action_run.clear_trigger_timeout = MagicMock() self.action_run._done(ActionRun.SUCCEEDED) assert self.action_run.clear_trigger_timeout.called def test_trigger_notify_clears_trigger_timeout(self): self.action_run.notify = MagicMock() self.action_run.triggered_by = [] self.action_run.clear_trigger_timeout = MagicMock() self.action_run.trigger_notify() assert self.action_run.clear_trigger_timeout.called class TestSSHActionRun: @pytest.fixture(autouse=True) def setup_action_run(self, output_path): self.action_runner = mock.create_autospec( actioncommand.NoActionRunnerFactory, ) self.command = "do command {actionname}" self.action_run = SSHActionRun( job_run_id="job_name.5", name="action_name", command_config=ActionCommandConfig(command=self.command), node=mock.create_autospec(node.Node), output_path=output_path, action_runner=self.action_runner, ) def test_start_node_error(self): def raise_error(c): raise node.Error("The error") self.action_run.node = mock.MagicMock() self.action_run.node.submit_command.side_effect = raise_error self.action_run.machine.transition("ready") assert not self.action_run.start() assert self.action_run.exit_status == -2 assert self.action_run.is_failed @mock.patch("tron.core.actionrun.filehandler", autospec=True) def test_build_action_command(self, mock_filehandler): self.action_run.watch = mock.MagicMock() attempt = self.action_run.create_attempt() serializer = mock_filehandler.OutputStreamSerializer.return_value action_command = self.action_run.build_action_command(attempt) assert action_command == self.action_run.action_command assert action_command == self.action_runner.create.return_value self.action_runner.create.assert_called_with( self.action_run.id, attempt.rendered_command, serializer, ) mock_filehandler.OutputStreamSerializer.assert_called_with( self.action_run.output_path, ) self.action_run.watch.assert_called_with(action_command) def test_handler_running(self): attempt = self.action_run.create_attempt() self.action_run.build_action_command(attempt) self.action_run.machine.transition("start") assert self.action_run.handler( self.action_run.action_command, ActionCommand.RUNNING, ) assert self.action_run.is_running def test_handler_failstart(self): attempt = self.action_run.create_attempt() self.action_run.build_action_command(attempt) assert self.action_run.handler( self.action_run.action_command, ActionCommand.FAILSTART, ) assert self.action_run.is_failed def test_handler_exiting_fail(self): attempt = self.action_run.create_attempt() self.action_run.build_action_command(attempt) self.action_run.action_command.exit_status = -1 self.action_run.machine.transition("start") assert self.action_run.handler( self.action_run.action_command, ActionCommand.EXITING, ) assert self.action_run.is_failed assert self.action_run.exit_status == -1 def test_handler_exiting_success(self): attempt = self.action_run.create_attempt() self.action_run.build_action_command(attempt) self.action_run.action_command.exit_status = 0 self.action_run.machine.transition("start") self.action_run.machine.transition("started") assert self.action_run.handler( self.action_run.action_command, ActionCommand.EXITING, ) assert self.action_run.is_succeeded assert self.action_run.exit_status == 0 def test_handler_exiting_failunknown(self): self.action_run.action_command = mock.create_autospec( actioncommand.ActionCommand, exit_status=None, ) self.action_run.machine.transition("start") self.action_run.machine.transition("started") assert self.action_run.handler( self.action_run.action_command, ActionCommand.EXITING, ) assert self.action_run.is_unknown assert self.action_run.exit_status is None assert self.action_run.end_time is not None def test_handler_unhandled(self): attempt = self.action_run.create_attempt() self.action_run.build_action_command(attempt) assert ( self.action_run.handler( self.action_run.action_command, ActionCommand.PENDING, ) is None ) assert self.action_run.is_scheduled def test_recover_no_action_runner(self): # Default setup has no action runner assert not self.action_run.recover() class TestSSHActionRunRecover: @pytest.fixture(autouse=True) def setup_action_run(self, output_path): self.action_runner = SubprocessActionRunnerFactory( status_path="/tmp/foo", exec_path="/bin/foo", ) self.command = "do command {actionname}" self.action_run = SSHActionRun( job_run_id="job_name.5", name="action_name", command_config=ActionCommandConfig(self.command), node=mock.create_autospec(node.Node), output_path=output_path, action_runner=self.action_runner, ) def test_recover_incorrect_state(self): # Should return falsy if not UNKNOWN. self.action_run.machine.state = ActionRun.FAILED assert not self.action_run.recover() def test_recover_action_runner(self): self.action_run.end_time = 1000 self.action_run.exit_status = 0 self.action_run.machine.state = ActionRun.UNKNOWN last_attempt = self.action_run.create_attempt() last_attempt.end_time = 1000 last_attempt.exit_status = 0 assert self.action_run.recover() assert self.action_run.machine.state == ActionRun.RUNNING assert self.action_run.end_time is None assert self.action_run.exit_status is None assert last_attempt.end_time is None assert last_attempt.exit_status is None self.action_run.node.submit_command.assert_called_once() # Check recovery command submit_args = self.action_run.node.submit_command.call_args[0] assert len(submit_args) == 1 recovery_command = submit_args[0] assert recovery_command.command == "/bin/foo/recover_batch.py /tmp/foo/job_name.5.action_name/status" assert recovery_command.start_time is not None # already started @mock.patch("tron.core.actionrun.reactor", autospec=True) def test_handler_exiting_failunknown(self, mock_reactor): self.action_run.action_command = mock.create_autospec( actioncommand.ActionCommand, exit_status=None, ) self.action_run.machine.transition("start") self.action_run.machine.transition("started") delay_deferred = self.action_run.handler( self.action_run.action_command, ActionCommand.EXITING, ) assert delay_deferred == mock_reactor.callLater.return_value assert self.action_run.is_running assert self.action_run.exit_status is None assert self.action_run.end_time is None call_args = mock_reactor.callLater.call_args[0] assert call_args[0] == INITIAL_RECOVER_DELAY assert call_args[1] == self.action_run.submit_recovery_command # Check recovery run recovery_run = call_args[2] assert "recovery" in recovery_run.name assert isinstance(recovery_run, SSHActionRun) # Recovery run should not be recovering itself, parent run handles its unknown status assert recovery_run.recover() is None # Check command recovery_command = call_args[3] assert recovery_command.command == "/bin/foo/recover_batch.py /tmp/foo/job_name.5.action_name/status" assert recovery_command.start_time is not None # already started @mock.patch("tron.core.actionrun.SSHActionRun.do_recover", autospec=True) @mock.patch("tron.core.actionrun.reactor", autospec=True) def test_handler_exiting_failunknown_max_retries(self, mock_reactor, mock_do_recover): self.action_run.action_command = mock.create_autospec( actioncommand.ActionCommand, exit_status=None, ) self.action_run.machine.transition("start") self.action_run.machine.transition("started") def exit_unknown(*args, **kwargs): self.action_run.handler( self.action_run.action_command, ActionCommand.EXITING, ) # Each time do_recover is called, end up exiting unknown again mock_do_recover.side_effect = exit_unknown # Start the cycle exit_unknown() assert mock_do_recover.call_count == MAX_RECOVER_TRIES last_call = mock_do_recover.call_args expected_delay = INITIAL_RECOVER_DELAY * (3 ** (MAX_RECOVER_TRIES - 1)) assert last_call == mock.call(self.action_run, delay=expected_delay) assert self.action_run.is_unknown assert self.action_run.exit_status is None assert self.action_run.end_time is not None class TestActionRunStateRestore: now = datetime.datetime(2012, 3, 14, 15, 19) @pytest.fixture(autouse=True) def setup_action_run(self, mock_current_time): self.parent_context = {} self.output_path = ["one", "two"] self.run_node = MagicMock() mock_current_time.return_value = self.now self.command_config = ActionCommandConfig( command="do {actionname}", cpus=1, ) self.action_config = mock.Mock(command_config=self.command_config) self.action_graph = actiongraph.ActionGraph( {"theaction": self.action_config}, {"theaction": set()}, {"theaction": set()}, ) @pytest.fixture def state_data(self): # State data with command config and retries. yield { "job_run_id": "theid", "action_name": "theaction", "node_name": "anode", "run_time": "the_run_time", "start_time": "start_time", "end_time": "end", "exit_status": 0, "attempts": [ dict( command_config=self.command_config.state_data, rendered_command="do theaction", start_time="start", end_time="end", exit_status=0, ), ], "state": "succeeded", } @pytest.fixture def state_data_old(self): # State data before command config and retries are separate. yield { "job_run_id": "theid", "action_name": "theaction", "node_name": "anode", "command": "do {actionname}", "start_time": "start_time", "end_time": "end", "state": "succeeded", } def test_from_state_old(self, state_data_old): state_data = state_data_old action_run = ActionRun.from_state( state_data, self.parent_context, list(self.output_path), self.run_node, self.action_graph, ) for key, value in state_data.items(): if key in ["state", "node_name"]: continue assert getattr(action_run, key) == value assert action_run.is_succeeded assert not action_run.is_cleanup assert action_run.output_path[:2] == self.output_path assert action_run.command_config.command == state_data["command"] assert action_run.command == state_data["command"] def test_from_state_old_with_mesos_task_id(self, state_data_old): state_data = state_data_old state_data["mesos_task_id"] = "task" action_run = ActionRun.from_state( state_data, self.parent_context, list(self.output_path), self.run_node, self.action_graph, ) for key, value in state_data.items(): if key in ["state", "node_name", "mesos_task_id"]: continue assert getattr(action_run, key) == value assert action_run.is_succeeded assert action_run.last_attempt.mesos_task_id == state_data["mesos_task_id"] def test_from_state_old_not_started(self, state_data_old): state_data = state_data_old state_data["start_time"] = None state_data["state"] = "scheduled" action_run = ActionRun.from_state( state_data, self.parent_context, list(self.output_path), self.run_node, self.action_graph, ) for key, value in state_data.items(): if key in ["state", "node_name"]: continue assert getattr(action_run, key) == value assert action_run.is_scheduled assert action_run.exit_statuses == [] assert len(action_run.attempts) == 0 def test_from_state_old_rendered_and_exited(self, state_data_old): state_data = state_data_old state_data["rendered_command"] = "do things theaction" state_data["exit_status"] = 0 action_run = ActionRun.from_state( state_data, self.parent_context, list(self.output_path), self.run_node, self.action_graph, ) for key, value in state_data.items(): if key in ["state", "node_name", "command", "rendered_command"]: continue assert getattr(action_run, key) == value assert action_run.is_succeeded assert action_run.exit_statuses == [0] assert action_run.command_config.command == state_data["command"] assert action_run.command == state_data["rendered_command"] def test_from_state_old_retries(self, state_data_old): state_data = state_data_old state_data["rendered_command"] = "do things theaction" state_data["exit_status"] = 0 state_data["exit_statuses"] = [1] action_run = ActionRun.from_state( state_data, self.parent_context, list(self.output_path), self.run_node, self.action_graph, ) for key, value in state_data.items(): if key in [ "state", "node_name", "command", "rendered_command", "exit_statuses", ]: continue assert getattr(action_run, key) == value assert action_run.is_succeeded assert action_run.exit_statuses == [1, 0] assert len(action_run.attempts) == 2 def test_from_state_running(self, state_data): state_data["state"] = "running" action_run = ActionRun.from_state( state_data, self.parent_context, self.output_path, self.run_node, self.action_graph, lambda: None, ) assert action_run.is_unknown def test_from_state_starting(self, state_data): state_data["state"] = "starting" action_run = ActionRun.from_state( state_data, self.parent_context, self.output_path, self.run_node, self.action_graph, lambda: None, ) assert action_run.is_unknown def test_from_state_queued(self, state_data): state_data["state"] = "queued" action_run = ActionRun.from_state( state_data, self.parent_context, self.output_path, self.run_node, self.action_graph, lambda: None, ) assert action_run.is_queued def test_from_state_no_node_name(self, state_data): del state_data["node_name"] action_run = ActionRun.from_state( state_data, self.parent_context, self.output_path, self.run_node, self.action_graph, lambda: None, ) assert action_run.node == self.run_node @mock.patch("tron.core.actionrun.node.NodePoolRepository", autospec=True) def test_from_state_with_node_exists(self, mock_store, state_data): ActionRun.from_state( state_data, self.parent_context, self.output_path, self.run_node, self.action_graph, lambda: None, ) mock_store.get_instance().get_node.assert_called_with( state_data["node_name"], self.run_node, ) def test_from_state_after_rendered_command(self, state_data): action_run = ActionRun.from_state( state_data, self.parent_context, self.output_path, self.run_node, self.action_graph, lambda: None, ) assert action_run.command_config == self.command_config assert len(action_run.attempts) == len(state_data["attempts"]) assert action_run.exit_statuses == [0] assert action_run.command == state_data["attempts"][-1]["rendered_command"] def test_from_state_action_config_gone(self, state_data): state_data["action_name"] = "old_action" action_run = ActionRun.from_state( state_data, self.parent_context, self.output_path, self.run_node, self.action_graph, lambda: None, ) assert action_run.command_config.command == "" assert action_run.command == state_data["attempts"][-1]["rendered_command"] class TestActionRunCollection: def _build_run(self, action): mock_node = mock.create_autospec(node.Node) return ActionRun( "id", action.name, mock_node, command_config=action.command_config, output_path=self.output_path, ) @pytest.fixture(autouse=True) def setup_runs(self, output_path): action_names = ["action_name", "second_name", "cleanup"] actions = [] for name in action_names: m = mock.Mock( name=name, required_actions=[], command_config=ActionCommandConfig(command="old"), ) m.name = name actions.append(m) self.action_graph = actiongraph.ActionGraph( {a.name: a for a in actions}, {"action_name": set(), "second_name": set(), "cleanup": set()}, {"action_name": set(), "second_name": set(), "cleanup": set()}, ) self.output_path = output_path self.command = "do command" self.action_runs = [self._build_run(action) for action in actions] self.run_map = {a.action_name: a for a in self.action_runs} self.run_map["cleanup"].is_cleanup = True self.collection = ActionRunCollection(self.action_graph, self.run_map) def test__init__(self): assert self.collection.action_graph == self.action_graph assert self.collection.run_map == self.run_map assert self.collection.proxy_action_runs_with_cleanup def test_action_runs_for_actions(self): m = MagicMock() m.name = "action_name" actions = [m] action_runs = self.collection.action_runs_for_actions(actions) assert list(action_runs) == self.action_runs[:1] def test_get_action_runs_with_cleanup(self): runs = self.collection.get_action_runs_with_cleanup() assert set(runs) == set(self.action_runs) def test_get_action_runs(self): runs = self.collection.get_action_runs() assert set(runs) == set(self.action_runs[:2]) def test_cleanup_action_run(self): assert self.action_runs[2] == self.collection.cleanup_action_run def test_update_action_config_no_changes(self): assert self.collection.update_action_config(self.action_graph) is False def test_update_action_config(self): # Latest config has 'new_name' instead of 'action_name' new_action_names = ["new_name", "second_name", "cleanup"] new_actions = [] for name in new_action_names: action = mock.Mock( name=name, required_actions=[], command_config=ActionCommandConfig(command="new"), ) action.name = name new_actions.append(action) new_action_graph = actiongraph.ActionGraph( {a.name: a for a in new_actions}, {"new_name": set(), "second_name": set(), "cleanup": set()}, {"new_name": set(), "second_name": set(), "cleanup": set()}, ) assert self.collection.update_action_config(new_action_graph) is True assert self.collection.action_graph != new_action_graph updated_action_runs = self.collection.action_runs_with_cleanup # Action names should be unchanged assert sorted(run.name for run in updated_action_runs) == sorted(run.name for run in self.action_runs) for run in updated_action_runs: if run.name == "action_name": assert run.command_config.command == "old" else: assert run.command_config.command == "new" def test_state_data(self): state_data = self.collection.state_data assert_length(state_data, len(self.action_runs[:2])) def test_cleanup_action_state_data(self): state_data = self.collection.cleanup_action_state_data assert state_data["action_name"] == "cleanup" def test_cleanup_action_state_data_no_cleanup_action(self): del self.collection.run_map["cleanup"] assert not self.collection.cleanup_action_state_data def test_get_startable_action_runs(self): action_runs = self.collection.get_startable_action_runs() assert set(action_runs) == set(self.action_runs[:2]) def test_get_startable_action_runs_none(self): self.collection.run_map.clear() action_runs = self.collection.get_startable_action_runs() assert set(action_runs) == set() def test_has_startable_action_runs(self): assert self.collection.has_startable_action_runs def test_has_startable_action_runs_false(self): self.collection.run_map.clear() assert not self.collection.has_startable_action_runs def test_is_complete_false(self): assert not self.collection.is_complete def test_is_complete_true(self): for action_run in self.collection.action_runs_with_cleanup: action_run.machine.state = ActionRun.SKIPPED assert self.collection.is_complete def test_is_done_false(self): assert not self.collection.is_done def test_is_done_false_because_of_running(self): action_run = self.collection.run_map["action_name"] action_run.machine.state = ActionRun.RUNNING assert not self.collection.is_done def test_is_done_true_because_blocked(self): self.run_map["action_name"].machine.state = ActionRun.FAILED self.run_map["second_name"].machine.state = ActionRun.WAITING autospec_method(self.collection._is_run_blocked) self.collection._is_run_blocked.return_value = True assert self.collection.is_done assert self.collection.is_failed self.collection._is_run_blocked.assert_called_with( self.run_map["second_name"], in_job_only=True, ) def test_is_done_true(self): for action_run in self.collection.action_runs_with_cleanup: action_run.machine.state = ActionRun.FAILED assert self.collection.is_done def test_is_failed_false_not_done(self): self.run_map["action_name"].machine.state = ActionRun.FAILED assert not self.collection.is_failed def test_is_failed_false_no_failed(self): for action_run in self.collection.action_runs_with_cleanup: action_run.machine.state = ActionRun.SUCCEEDED assert not self.collection.is_failed def test_is_failed_true(self): for action_run in self.collection.action_runs_with_cleanup: action_run.machine.state = ActionRun.FAILED assert self.collection.is_failed def test__getattr__(self): assert self.collection.is_scheduled assert not self.collection.is_cancelled assert not self.collection.is_running assert self.collection.ready() def test__str__(self): self.collection._is_run_blocked = lambda r: r.action_name != "cleanup" expected = [ "ActionRunCollection", "second_name(scheduled:blocked)", "action_name(scheduled:blocked)", "cleanup(scheduled)", ] for expectation in expected: assert expectation in str(self.collection) def test_end_time(self): max_end_time = datetime.datetime(2013, 6, 15) self.run_map["action_name"].machine.state = ActionRun.FAILED self.run_map["action_name"].end_time = datetime.datetime(2013, 5, 12) self.run_map["second_name"].machine.state = ActionRun.SUCCEEDED self.run_map["second_name"].end_time = max_end_time assert self.collection.end_time == max_end_time def test_end_time_not_done(self): self.run_map["action_name"].end_time = datetime.datetime(2013, 5, 12) self.run_map["action_name"].machine.state = ActionRun.FAILED self.run_map["second_name"].end_time = None self.run_map["second_name"].machine.state = ActionRun.RUNNING assert self.collection.end_time is None def test_end_time_not_started(self): assert self.collection.end_time is None class TestActionRunCollectionIsRunBlocked: def _build_run(self, name): mock_node = mock.create_autospec(node.Node) return ActionRun( "id", name, mock_node, self.command_config, output_path=self.output_path, ) @pytest.fixture(autouse=True) def setup_collection(self, output_path): action_names = ["action_name", "second_name", "cleanup"] actions = [] for name in action_names: m = MagicMock() m.name = name actions.append(m) self.second_act = actions[1] action_map = {a.name: a for a in actions} self.action_graph = actiongraph.ActionGraph( action_map, {"action_name": set(), "second_name": {"action_name"}, "cleanup": set()}, {"action_name": set(), "second_name": set(), "cleanup": set()}, ) self.output_path = output_path self.command_config = ActionCommandConfig(command="do command") self.action_runs = [self._build_run(name) for name in action_names] self.run_map = {a.action_name: a for a in self.action_runs} self.run_map["cleanup"].is_cleanup = True self.collection = ActionRunCollection(self.action_graph, self.run_map) def test_is_run_blocked_no_required_actions(self): assert not self.collection._is_run_blocked(self.run_map["action_name"]) def test_is_run_blocked_completed_run(self): self.run_map["second_name"].machine.state = ActionRun.FAILED assert not self.collection._is_run_blocked(self.run_map["second_name"]) self.run_map["second_name"].machine.state = ActionRun.RUNNING assert not self.collection._is_run_blocked(self.run_map["second_name"]) def test_is_run_blocked_required_actions_completed(self): self.run_map["action_name"].machine.state = ActionRun.SKIPPED assert not self.collection._is_run_blocked(self.run_map["second_name"]) def test_is_run_blocked_required_actions_blocked(self): third_act = MagicMock() third_act.name = "third_act" self.action_graph.action_map["third_act"] = third_act self.action_graph.required_actions["third_act"] = {self.second_act.name} self.run_map["third_act"] = self._build_run("third_act") self.run_map["action_name"].machine.state = ActionRun.FAILED assert self.collection._is_run_blocked(self.run_map["third_act"]) def test_is_run_blocked_required_actions_scheduled(self): self.run_map["action_name"].machine.state = ActionRun.SCHEDULED assert self.collection._is_run_blocked(self.run_map["second_name"]) def test_is_run_blocked_required_actions_starting(self): self.run_map["action_name"].machine.state = ActionRun.STARTING assert self.collection._is_run_blocked(self.run_map["second_name"]) def test_is_run_blocked_required_actions_waiting(self): self.run_map["action_name"].machine.state = ActionRun.WAITING assert self.collection._is_run_blocked(self.run_map["second_name"]) def test_is_run_blocked_required_actions_failed(self): self.run_map["action_name"].machine.state = ActionRun.FAILED assert self.collection._is_run_blocked(self.run_map["second_name"]) def test_is_run_blocked_required_actions_missing(self): del self.run_map["action_name"] assert not self.collection._is_run_blocked(self.run_map["second_name"]) def test_is_run_blocked_in_job_only(self): self.run_map["action_name"].machine.state = ActionRun.SKIPPED self.run_map["second_name"].triggered_by = ["trigger"] assert not self.collection._is_run_blocked(self.run_map["second_name"], in_job_only=True) assert self.collection._is_run_blocked(self.run_map["second_name"], in_job_only=False) class TestMesosActionRun: @pytest.fixture(autouse=True) def setup_action_run(self): self.output_path = mock.MagicMock() self.command = "do the command" self.extra_volumes = [ConfigVolume("/mnt/foo", "/mnt/foo", "RO")] self.constraints = [ConfigConstraint("an attr", "an op", "a val")] self.docker_parameters = [ConfigParameter("init", "true")] self.other_task_kwargs = { "cpus": 1, "mem": 50, "disk": 42, "docker_image": "container:v2", "env": { "TESTING": "true", "TRON_JOB_NAMESPACE": "mynamespace", "TRON_JOB_NAME": "myjob", "TRON_RUN_NUM": "42", "TRON_ACTION": "action_name", }, } command_config = ActionCommandConfig( command=self.command, extra_volumes=self.extra_volumes, constraints=self.constraints, docker_parameters=self.docker_parameters, **self.other_task_kwargs, ) self.action_run = MesosActionRun( job_run_id="mynamespace.myjob.42", name="action_name", command_config=command_config, node=mock.create_autospec(node.Node), output_path=self.output_path, executor=ExecutorTypes.mesos.value, ) @mock.patch("tron.core.actionrun.filehandler", autospec=True) @mock.patch("tron.core.actionrun.MesosClusterRepository", autospec=True) def test_submit_command(self, mock_cluster_repo, mock_filehandler): serializer = mock_filehandler.OutputStreamSerializer.return_value # submit_command should add a new attempt self.action_run.attempts = [ ActionRunAttempt( command_config=self.action_run.command_config, rendered_command=self.command, mesos_task_id="last_attempt", ), ] with mock.patch.object( self.action_run, "watch", autospec=True, ) as mock_watch: new_attempt = self.action_run.create_attempt() self.action_run.submit_command(new_attempt) mock_get_cluster = mock_cluster_repo.get_cluster mock_get_cluster.assert_called_once_with() mock_get_cluster.return_value.create_task.assert_called_once_with( action_run_id=self.action_run.id, command=self.command, serializer=serializer, task_id=None, extra_volumes=[e._asdict() for e in self.extra_volumes], constraints=[["an attr", "an op", "a val"]], docker_parameters=[{"key": "init", "value": "true"}], **self.other_task_kwargs, ) task = mock_get_cluster.return_value.create_task.return_value mock_get_cluster.return_value.submit.assert_called_once_with(task) mock_watch.assert_called_once_with(task) assert self.action_run.last_attempt.mesos_task_id == task.get_mesos_id.return_value mock_filehandler.OutputStreamSerializer.assert_called_with( self.action_run.output_path, ) @mock.patch("tron.core.actionrun.filehandler", autospec=True) @mock.patch("tron.core.actionrun.MesosClusterRepository", autospec=True) def test_submit_command_task_none( self, mock_cluster_repo, mock_filehandler, ): # Task is None if Mesos is disabled mock_get_cluster = mock_cluster_repo.get_cluster mock_get_cluster.return_value.create_task.return_value = None new_attempt = self.action_run.create_attempt() self.action_run.submit_command(new_attempt) mock_get_cluster.assert_called_once_with() assert mock_get_cluster.return_value.submit.call_count == 0 assert self.action_run.is_failed @mock.patch("tron.core.actionrun.filehandler", autospec=True) @mock.patch("tron.core.actionrun.MesosClusterRepository", autospec=True) def test_recover(self, mock_cluster_repo, mock_filehandler): self.action_run.machine.state = ActionRun.UNKNOWN self.action_run.end_time = 1000 self.action_run.exit_status = 0 last_attempt = self.action_run.create_attempt() last_attempt.mesos_task_id = "my_mesos_id" last_attempt.end_time = 1000 last_attempt.exit_status = 0 serializer = mock_filehandler.OutputStreamSerializer.return_value with mock.patch.object( self.action_run, "watch", autospec=True, ) as mock_watch: assert self.action_run.recover() mock_get_cluster = mock_cluster_repo.get_cluster mock_get_cluster.assert_called_once_with() mock_get_cluster.return_value.create_task.assert_called_once_with( action_run_id=self.action_run.id, command=self.command, serializer=serializer, task_id="my_mesos_id", extra_volumes=[e._asdict() for e in self.extra_volumes], constraints=[["an attr", "an op", "a val"]], docker_parameters=[{"key": "init", "value": "true"}], **self.other_task_kwargs, ) task = mock_get_cluster.return_value.create_task.return_value mock_get_cluster.return_value.recover.assert_called_once_with(task) mock_watch.assert_called_once_with(task) assert self.action_run.is_running assert self.action_run.end_time is None assert self.action_run.exit_status is None assert last_attempt.end_time is None assert last_attempt.exit_status is None mock_filehandler.OutputStreamSerializer.assert_called_with( self.action_run.output_path, ) @mock.patch("tron.core.actionrun.filehandler", autospec=True) @mock.patch("tron.core.actionrun.MesosClusterRepository", autospec=True) def test_recover_done_no_change(self, mock_cluster_repo, mock_filehandler): self.action_run.machine.state = ActionRun.SUCCEEDED last_attempt = self.action_run.create_attempt() last_attempt.mesos_task_id = "my_mesos_id" assert not self.action_run.recover() assert mock_cluster_repo.get_cluster.call_count == 0 assert self.action_run.is_succeeded @mock.patch("tron.core.actionrun.filehandler", autospec=True) @mock.patch("tron.core.actionrun.MesosClusterRepository", autospec=True) def test_recover_no_mesos_task_id( self, mock_cluster_repo, mock_filehandler, ): self.action_run.machine.state = ActionRun.UNKNOWN last_attempt = self.action_run.create_attempt() last_attempt.mesos_task_id = None assert not self.action_run.recover() assert mock_cluster_repo.get_cluster.call_count == 0 assert self.action_run.is_unknown assert self.action_run.end_time is not None @mock.patch("tron.core.actionrun.filehandler", autospec=True) @mock.patch("tron.core.actionrun.MesosClusterRepository", autospec=True) def test_recover_task_none(self, mock_cluster_repo, mock_filehandler): self.action_run.machine.state = ActionRun.UNKNOWN last_attempt = self.action_run.create_attempt() last_attempt.mesos_task_id = "my_mesos_id" # Task is None if Mesos is disabled mock_get_cluster = mock_cluster_repo.get_cluster mock_get_cluster.return_value.create_task.return_value = None assert not self.action_run.recover() mock_get_cluster.assert_called_once_with() assert self.action_run.is_unknown assert mock_get_cluster.return_value.recover.call_count == 0 assert self.action_run.end_time is not None @mock.patch("tron.core.actionrun.MesosClusterRepository", autospec=True) def test_kill_task(self, mock_cluster_repo): mock_get_cluster = mock_cluster_repo.get_cluster last_attempt = self.action_run.create_attempt() last_attempt.mesos_task_id = "fake_task_id" self.action_run.machine.state = ActionRun.RUNNING self.action_run.kill() mock_get_cluster.return_value.kill.assert_called_once_with( last_attempt.mesos_task_id, ) @mock.patch("tron.core.actionrun.MesosClusterRepository", autospec=True) def test_kill_task_no_task_id(self, mock_cluster_repo): self.action_run.machine.state = ActionRun.RUNNING self.action_run.create_attempt() error_message = self.action_run.kill() assert error_message == "Error: Can't find task id for the action." @mock.patch("tron.core.actionrun.MesosClusterRepository", autospec=True) def test_stop_task(self, mock_cluster_repo): mock_get_cluster = mock_cluster_repo.get_cluster last_attempt = self.action_run.create_attempt() last_attempt.mesos_task_id = "fake_task_id" self.action_run.machine.state = ActionRun.RUNNING self.action_run.stop() mock_get_cluster.return_value.kill.assert_called_once_with( last_attempt.mesos_task_id, ) @mock.patch("tron.core.actionrun.MesosClusterRepository", autospec=True) def test_stop_task_no_task_id(self, mock_cluster_repo): self.action_run.machine.state = ActionRun.RUNNING self.action_run.create_attempt() error_message = self.action_run.stop() assert error_message == "Error: Can't find task id for the action." def test_handler_exiting_unknown(self): self.action_run.action_command = mock.create_autospec( actioncommand.ActionCommand, exit_status=None, ) self.action_run.machine.transition("start") self.action_run.machine.transition("started") assert self.action_run.handler( self.action_run.action_command, ActionCommand.EXITING, ) assert self.action_run.is_unknown assert self.action_run.exit_status is None assert self.action_run.end_time is not None def test_handler_exiting_unknown_retry(self): self.action_run.action_command = mock.create_autospec( actioncommand.ActionCommand, exit_status=None, ) self.action_run.retries_remaining = 1 self.action_run.start = mock.Mock() self.action_run.machine.transition("start") self.action_run.machine.transition("started") assert self.action_run.handler( self.action_run.action_command, ActionCommand.EXITING, ) assert self.action_run.retries_remaining == 0 assert not self.action_run.is_unknown assert self.action_run.start.call_count == 1 def test_handler_exiting_failstart_failed(self): self.action_run.action_command = mock.create_autospec( actioncommand.ActionCommand, exit_status=1, ) self.action_run.machine.transition("start") assert self.action_run.handler( self.action_run.action_command, ActionCommand.FAILSTART, ) assert self.action_run.is_failed class TestKubernetesActionRun: @pytest.fixture def mock_k8s_action_run(self): command_config = ActionCommandConfig( command="mock_command", extra_volumes=set(), constraints=set(), docker_parameters=set(), cpus=1, mem=50, disk=42, docker_image="container:v2", env={ "TESTING": "true", "TRON_JOB_NAMESPACE": "mock_namespace", "TRON_JOB_NAME": "mock_job", "TRON_RUN_NUM": "42", "TRON_ACTION": "mock_action_name", }, labels={ "tron.yelp.com/run_num": "42", }, ) return KubernetesActionRun( job_run_id="mock_namespace.mock_job.42", name="mock_action_name", command_config=command_config, node=mock.create_autospec(node.Node), output_path=mock.create_autospec(filehandler.OutputPath), executor=ExecutorTypes.kubernetes.value, ) def test_k8s_handler_exiting_unknown(self, mock_k8s_action_run): mock_k8s_action_run.action_command = mock.create_autospec( actioncommand.ActionCommand, exit_status=None, ) mock_k8s_action_run.machine.transition("start") mock_k8s_action_run.machine.transition("started") assert mock_k8s_action_run.handler( mock_k8s_action_run.action_command, ActionCommand.EXITING, ) assert mock_k8s_action_run.is_unknown assert mock_k8s_action_run.exit_status is None assert mock_k8s_action_run.end_time is not None def test_handler_exiting_unknown_retry(self, mock_k8s_action_run): mock_k8s_action_run.action_command = mock.create_autospec( actioncommand.ActionCommand, exit_status=None, ) mock_k8s_action_run.retries_remaining = 1 mock_k8s_action_run.start = mock.Mock() mock_k8s_action_run.machine.transition("start") mock_k8s_action_run.machine.transition("started") assert mock_k8s_action_run.handler( mock_k8s_action_run.action_command, ActionCommand.EXITING, ) assert mock_k8s_action_run.retries_remaining == 0 assert not mock_k8s_action_run.is_unknown assert mock_k8s_action_run.start.call_count == 1 def test_handler_exiting_failstart_failed(self, mock_k8s_action_run): mock_k8s_action_run.action_command = mock.create_autospec( actioncommand.ActionCommand, exit_status=1, ) mock_k8s_action_run.machine.transition("start") assert mock_k8s_action_run.handler( mock_k8s_action_run.action_command, ActionCommand.FAILSTART, ) assert mock_k8s_action_run.is_failed @mock.patch("tron.core.actionrun.filehandler", autospec=True) @mock.patch("tron.core.actionrun.KubernetesClusterRepository", autospec=True) def test_recover(self, mock_cluster_repo, mock_filehandler, mock_k8s_action_run): mock_k8s_action_run.machine.state = ActionRun.UNKNOWN mock_k8s_action_run.end_time = 1000 mock_k8s_action_run.exit_status = 0 last_attempt = mock_k8s_action_run.create_attempt() last_attempt.kubernetes_task_id = "test-k8s-task-id" last_attempt.end_time = 1000 last_attempt.exit_status = 0 serializer = mock_filehandler.OutputStreamSerializer.return_value with mock.patch.object( mock_k8s_action_run, "watch", autospec=True, ) as mock_watch: assert mock_k8s_action_run.recover() mock_get_cluster = mock_cluster_repo.get_cluster mock_get_cluster.assert_called_once_with() mock_get_cluster.return_value.create_task.assert_called_once_with( action_run_id=mock_k8s_action_run.id, command=last_attempt.rendered_command, cpus=mock_k8s_action_run.command_config.cpus, mem=mock_k8s_action_run.command_config.mem, disk=mock_k8s_action_run.command_config.disk, docker_image=mock_k8s_action_run.command_config.docker_image, env=mock.ANY, secret_env=mock_k8s_action_run.command_config.secret_env, field_selector_env=mock_k8s_action_run.command_config.field_selector_env, serializer=serializer, volumes=mock_k8s_action_run.command_config.extra_volumes, secret_volumes=mock_k8s_action_run.command_config.secret_volumes, projected_sa_volumes=mock_k8s_action_run.command_config.projected_sa_volumes, cap_add=mock_k8s_action_run.command_config.cap_add, cap_drop=mock_k8s_action_run.command_config.cap_drop, task_id=last_attempt.kubernetes_task_id, node_selectors=mock_k8s_action_run.command_config.node_selectors, node_affinities=mock_k8s_action_run.command_config.node_affinities, topology_spread_constraints=mock_k8s_action_run.command_config.topology_spread_constraints, pod_labels={ "tron.yelp.com/run_num": "42", "tron.yelp.com/attempt_number": "0", }, pod_annotations=mock_k8s_action_run.command_config.annotations, service_account_name=mock_k8s_action_run.command_config.service_account_name, ports=mock_k8s_action_run.command_config.ports, ) task = mock_get_cluster.return_value.create_task.return_value mock_get_cluster.return_value.recover.assert_called_once_with(task) mock_watch.assert_called_once_with(task) assert mock_k8s_action_run.is_running assert mock_k8s_action_run.end_time is None assert mock_k8s_action_run.exit_status is None assert last_attempt.end_time is None assert last_attempt.exit_status is None mock_filehandler.OutputStreamSerializer.assert_called_with( mock_k8s_action_run.output_path, ) @mock.patch("tron.core.actionrun.filehandler", autospec=True) @mock.patch("tron.core.actionrun.KubernetesClusterRepository", autospec=True) def test_recover_done_no_change( self, mock_cluster_repo, mock_filehandler, mock_k8s_action_run, ): mock_k8s_action_run.machine.state = ActionRun.SUCCEEDED last_attempt = mock_k8s_action_run.create_attempt() last_attempt.kubernetes_task_ic = "test-kubernetes-task-id" assert not mock_k8s_action_run.recover() assert mock_cluster_repo.get_cluster.call_count != 0 assert mock_k8s_action_run.is_succeeded @mock.patch("tron.core.actionrun.filehandler", autospec=True) @mock.patch("tron.core.actionrun.KubernetesClusterRepository", autospec=True) def test_recover_no_k8s_task_id( self, mock_cluster_repo, mock_filehandler, mock_k8s_action_run, ): print(f"cluster: {type(mock_cluster_repo)} filehand: {type(mock_filehandler)} ar: {type(mock_k8s_action_run)}") mock_k8s_action_run.machine.state = ActionRun.UNKNOWN last_attempt = mock_k8s_action_run.create_attempt() last_attempt.mesos_task_id = None assert not mock_k8s_action_run.recover() assert mock_k8s_action_run.is_unknown assert mock_k8s_action_run.end_time is not None @mock.patch("tron.core.actionrun.filehandler", autospec=True) @mock.patch("tron.core.actionrun.KubernetesClusterRepository", autospec=True) def test_recover_task_none(self, mock_cluster_repo, mock_filehandler, mock_k8s_action_run): mock_k8s_action_run.machine.state = ActionRun.UNKNOWN last_attempt = mock_k8s_action_run.create_attempt() last_attempt.kubernetes_task_id = "test-kubernetes-task-id" # Task is None e.g. if Kubernetes is disabled mock_get_cluster = mock_cluster_repo.get_cluster mock_get_cluster.return_value.create_task.return_value = None assert not mock_k8s_action_run.recover() mock_get_cluster.assert_called_once_with() assert mock_k8s_action_run.is_unknown assert mock_get_cluster.return_value.recover.call_count == 0 assert mock_k8s_action_run.end_time is not None @mock.patch("tron.core.actionrun.KubernetesClusterRepository", autospec=True) def test_kill_task_k8s(self, mock_cluster_repo, mock_k8s_action_run): mock_get_cluster = mock_cluster_repo.get_cluster last_attempt = mock_k8s_action_run.create_attempt() last_attempt.kubernetes_task_id = "fake_task_id" mock_k8s_action_run.machine.state = ActionRun.RUNNING mock_k8s_action_run.kill() mock_get_cluster.return_value.kill.assert_called_once_with(last_attempt.kubernetes_task_id) @mock.patch("tron.core.actionrun.KubernetesClusterRepository", autospec=True) def test_kill_task_no_task_id_k8s(self, mock_cluster_repo, mock_k8s_action_run): mock_k8s_action_run.machine.state = ActionRun.RUNNING mock_k8s_action_run.create_attempt() error_message = mock_k8s_action_run.kill() assert error_message == "Error: Can't find task id for the action." @mock.patch("tron.core.actionrun.KubernetesClusterRepository", autospec=True) def test_stop_task_k8s(self, mock_cluster_repo, mock_k8s_action_run): mock_get_cluster = mock_cluster_repo.get_cluster last_attempt = mock_k8s_action_run.create_attempt() last_attempt.kubernetes_task_id = "fake_task_id" mock_k8s_action_run.machine.state = ActionRun.RUNNING mock_k8s_action_run.stop() mock_get_cluster.return_value.kill.assert_called_once_with(last_attempt.kubernetes_task_id) @mock.patch("tron.core.actionrun.KubernetesClusterRepository", autospec=True) def test_stop_task_no_task_id_k8s(self, mock_cluster_repo, mock_k8s_action_run): mock_k8s_action_run.machine.state = ActionRun.RUNNING mock_k8s_action_run.create_attempt() error_message = mock_k8s_action_run.stop() assert error_message == "Error: Can't find task id for the action." @mock.patch("tron.core.actionrun.KubernetesClusterRepository", autospec=True) def test_non_retryable_exit(self, mock_cluster_repo, mock_k8s_action_run): mock_cluster = mock.Mock() mock_cluster.non_retryable_exit_codes = [13] mock_cluster_repo.get_cluster.return_value = mock_cluster mock_k8s_action_run.action_command = mock.create_autospec( actioncommand.ActionCommand, exit_status=13, ) mock_k8s_action_run.retries_remaining = 5 mock_k8s_action_run.machine.transition("start") mock_k8s_action_run.machine.transition("started") assert mock_k8s_action_run.handler( mock_k8s_action_run.action_command, ActionCommand.EXITING, ) assert mock_k8s_action_run.retries_remaining == 0 assert mock_k8s_action_run.is_unknown @mock.patch("tron.core.actionrun.KubernetesClusterRepository", autospec=True) def test_retryable_exit(self, mock_cluster_repo, mock_k8s_action_run): mock_cluster = mock.Mock() mock_cluster.non_retryable_exit_codes = [-12] mock_cluster_repo.get_cluster.return_value = mock_cluster mock_k8s_action_run.retries_remaining = 5 mock_k8s_action_run.start = mock.Mock() mock_k8s_action_run._exit_unsuccessful(13) assert mock_k8s_action_run.retries_remaining == 4 @mock.patch("tron.core.actionrun.filehandler", autospec=True) @mock.patch("tron.core.actionrun.KubernetesClusterRepository", autospec=True) def test_submit_command_first_attempt_labels(self, mock_cluster_repo, mock_filehandler, mock_k8s_action_run): with mock.patch.object(mock_k8s_action_run, "watch", autospec=True): new_attempt = mock_k8s_action_run.create_attempt() mock_k8s_action_run.submit_command(new_attempt) create_task_kwargs = mock_cluster_repo.get_cluster.return_value.create_task.call_args[1] assert create_task_kwargs["pod_labels"]["tron.yelp.com/attempt_number"] == "0" @mock.patch("tron.core.actionrun.filehandler", autospec=True) @mock.patch("tron.core.actionrun.KubernetesClusterRepository", autospec=True) def test_submit_command_retry_attempt_labels(self, mock_cluster_repo, mock_filehandler, mock_k8s_action_run): mock_k8s_action_run.attempts = [ ActionRunAttempt( command_config=mock_k8s_action_run.command_config, rendered_command="mock_command", ), ActionRunAttempt( command_config=mock_k8s_action_run.command_config, rendered_command="mock_command", ), ] with mock.patch.object(mock_k8s_action_run, "watch", autospec=True): new_attempt = mock_k8s_action_run.create_attempt() mock_k8s_action_run.submit_command(new_attempt) create_task_kwargs = mock_cluster_repo.get_cluster.return_value.create_task.call_args[1] assert create_task_kwargs["pod_labels"]["tron.yelp.com/attempt_number"] == "2" @mock.patch("tron.core.actionrun.filehandler", autospec=True) @mock.patch("tron.core.actionrun.KubernetesClusterRepository", autospec=True) def test_recover_retry_attempt_labels(self, mock_cluster_repo, mock_filehandler, mock_k8s_action_run): mock_k8s_action_run.attempts = [ ActionRunAttempt( command_config=mock_k8s_action_run.command_config, rendered_command="mock_command", ), ActionRunAttempt( command_config=mock_k8s_action_run.command_config, rendered_command="mock_command", ), ] mock_k8s_action_run.machine.state = ActionRun.UNKNOWN last_attempt = mock_k8s_action_run.create_attempt() last_attempt.kubernetes_task_id = "test-k8s-task-id" with mock.patch.object(mock_k8s_action_run, "watch", autospec=True): assert mock_k8s_action_run.recover() create_task_kwargs = mock_cluster_repo.get_cluster.return_value.create_task.call_args[1] assert create_task_kwargs["pod_labels"]["tron.yelp.com/attempt_number"] == "2" ================================================ FILE: tests/core/job_collection_test.py ================================================ from unittest import mock from testifycompat import setup from testifycompat import TestCase from tests.testingutils import autospec_method from tron.core.job import Job from tron.core.job_collection import JobCollection from tron.core.job_scheduler import JobScheduler from tron.core.job_scheduler import JobSchedulerFactory class TestJobCollection(TestCase): @setup def setup_collection(self): self.collection = JobCollection() def test_update_from_config(self): autospec_method(self.collection.jobs.filter_by_name) autospec_method(self.collection.add) factory = mock.create_autospec(JobSchedulerFactory) job_configs = {"a": mock.Mock(), "b": mock.Mock()} result = self.collection.update_from_config(job_configs, factory, True) result = list(result) assert len(result) == len(job_configs) self.collection.jobs.filter_by_name.assert_called_with(job_configs) expected_calls = [mock.call(v) for v in job_configs.values()] assert factory.build.call_args_list == expected_calls assert self.collection.add.call_count == 2 job_schedulers = [call[1][0] for call in self.collection.add.mock_calls[::2]] for job_scheduler in job_schedulers: job_scheduler.schedule.assert_called_with() job_scheduler.get_job.assert_called_with() def test_update_from_config_reconfigure_one_namespace(self): autospec_method(self.collection.jobs.filter_by_name) autospec_method(self.collection.add) factory = mock.create_autospec(JobSchedulerFactory) job_configs = { "a.foo": mock.Mock(namespace="a"), "b.foo": mock.Mock(namespace="b"), } result = self.collection.update_from_config(job_configs, factory, True, namespace_to_reconfigure="a") result = list(result) assert len(result) == 1 self.collection.jobs.filter_by_name.assert_called_with(job_configs) expected_calls = [mock.call(job_configs["a.foo"])] assert factory.build.call_args_list == expected_calls assert self.collection.add.call_count == 1 job_schedulers = [call[1][0] for call in self.collection.add.mock_calls[::2]] for job_scheduler in job_schedulers: job_scheduler.schedule.assert_called_with() job_scheduler.get_job.assert_called_with() def test_move_running_job(self): with mock.patch( "tron.core.job_collection.JobCollection.get_by_name", autospec=None, ) as mock_scheduler: mock_scheduler.return_value.get_job.return_value.status = Job.STATUS_RUNNING result = self.collection.move("old.test", "new.test") assert "Job is still running." in result def test_move(self): with mock.patch( "tron.core.job_collection.JobCollection.get_by_name", autospec=None, ) as mock_scheduler: mock_scheduler.return_value.get_job.return_value.status = Job.STATUS_ENABLED mock_scheduler.get_name.return_value = "old.test" self.collection.add(mock_scheduler) result = self.collection.move("old.test", "new.test") assert "succeeded" in result def test_update(self): mock_scheduler = mock.create_autospec(JobScheduler) existing_scheduler = mock.create_autospec(JobScheduler) autospec_method( self.collection.get_by_name, return_value=existing_scheduler, ) assert self.collection.update(mock_scheduler) self.collection.get_by_name.assert_called_with( mock_scheduler.get_name(), ) existing_scheduler.update_from_job_scheduler.assert_called_with( mock_scheduler, ) existing_scheduler.schedule_reconfigured.assert_called_with() ================================================ FILE: tests/core/job_scheduler_test.py ================================================ import datetime from unittest import mock from testifycompat import assert_equal from testifycompat import setup from testifycompat import TestCase from tests import testingutils from tests.assertions import assert_length from tron import actioncommand from tron.core import job from tron.core.actionrun import ActionRun from tron.core.job_scheduler import JobScheduler from tron.core.job_scheduler import JobSchedulerFactory class TestJobSchedulerGetRunsToSchedule(TestCase): @setup def setup_job(self): self.scheduler = mock.Mock() run_collection = mock.Mock(has_pending=False) node_pool = mock.Mock() self.job = job.Job( "jobname", self.scheduler, run_collection=run_collection, node_pool=node_pool, ) self.job_scheduler = JobScheduler(self.job) self.job.runs.get_pending.return_value = False self.scheduler.queue_overlapping = True def test_get_runs_to_schedule_with_pending(self): self.scheduler.queue_overlapping = False self.job.runs.has_pending = True job_runs = self.job_scheduler.get_runs_to_schedule(None) assert_length(job_runs, 0) def test_get_runs_to_schedule_guess(self): job_runs = list(self.job_scheduler.get_runs_to_schedule(None)) assert self.job.scheduler.next_run_time.call_args_list == [mock.call(None)] assert_length(job_runs, 1) # This should return a JobRun which has the job attached as an observer job_runs[0].attach.assert_any_call(True, self.job) def test_get_runs_to_schedule_given(self): now = datetime.datetime.now() job_runs = list(self.job_scheduler.get_runs_to_schedule(now)) assert self.job.scheduler.next_run_time.call_count == 0 assert_length(job_runs, 1) # This should return a JobRun which has the job attached as an observer job_runs[0].attach.assert_any_call(True, self.job) class JobSchedulerManualStartTestCase(testingutils.MockTimeTestCase): now = datetime.datetime.now() @setup def setup_job(self): self.scheduler = mock.Mock() run_collection = mock.Mock() node_pool = mock.Mock() self.job = job.Job( "jobname", self.scheduler, run_collection=run_collection, node_pool=node_pool, ) self.job_scheduler = JobScheduler(self.job) self.manual_run = mock.Mock() self.job.build_new_runs = mock.Mock(return_value=[self.manual_run]) def test_manual_start(self): manual_runs = self.job_scheduler.manual_start() self.job.build_new_runs.assert_called_with(self.now, manual=True) assert_length(manual_runs, 1) self.manual_run.start.assert_called_once_with() def test_manual_start_default_with_timezone(self): self.job.time_zone = mock.Mock() with mock.patch( "tron.core.job_scheduler.timeutils.current_time", autospec=True, ) as mock_current: manual_runs = self.job_scheduler.manual_start() mock_current.assert_called_with(tz=self.job.time_zone) self.job.build_new_runs.assert_called_with( mock_current.return_value, manual=True, ) assert_length(manual_runs, 1) self.manual_run.start.assert_called_once_with() def test_manual_start_with_run_time(self): run_time = datetime.datetime(2012, 3, 14, 15, 9, 26) manual_runs = self.job_scheduler.manual_start(run_time) self.job.build_new_runs.assert_called_with(run_time, manual=True) assert_length(manual_runs, 1) self.manual_run.start.assert_called_once_with() class TestJobSchedulerSchedule(TestCase): @setup def setup_job(self): self.scheduler = mock.Mock(autospec=True) self.scheduler.next_run_time.return_value = 0 mock_run = mock.Mock() mock_run.seconds_until_run_time.return_value = 0 run_collection = mock.Mock( has_pending=False, autospec=True, return_value=[mock_run], ) mock_build_new_run = mock.Mock() run_collection.build_new_run.return_value = mock_build_new_run mock_build_new_run.seconds_until_run_time.return_value = 0 node_pool = mock.Mock() self.job = job.Job( name="jobname", scheduler=self.scheduler, run_collection=run_collection, node_pool=node_pool, ) self.job_scheduler = JobScheduler(self.job) self.original_build_new_runs = self.job.build_new_runs self.job.build_new_runs = mock.Mock(return_value=[mock_run]) @mock.patch("tron.core.job_scheduler.reactor", autospec=True) def test_enable(self, reactor): self.job.enabled = False self.job_scheduler.enable() assert self.job.enabled assert_length(reactor.callLater.mock_calls, 1) @mock.patch("tron.core.job_scheduler.reactor", autospec=True) def test_enable_noop(self, reactor): self.job.enabled = True self.job_scheduler.enable() assert self.job.enabled assert_length(reactor.callLater.mock_calls, 0) @mock.patch("tron.core.job_scheduler.reactor", autospec=True) def test_schedule(self, reactor): self.job.build_new_runs = self.original_build_new_runs self.job_scheduler.schedule() assert reactor.callLater.call_count == 1 # Args passed to callLater call_args = reactor.callLater.mock_calls[0][1] assert_equal(call_args[1], self.job_scheduler.run_job) secs = call_args[0] run = call_args[2] run.seconds_until_run_time.assert_called_with() # Assert that we use the seconds we get from the run to schedule assert_equal(run.seconds_until_run_time.return_value, secs) @mock.patch("tron.core.job_scheduler.reactor", autospec=True) def test_schedule_disabled_job(self, reactor): self.job.enabled = False self.job_scheduler.schedule() assert reactor.callLater.call_count == 0 @mock.patch("tron.core.job_scheduler.reactor", autospec=True) def test_handle_job_events_no_schedule_on_complete(self, reactor): self.job_scheduler.run_job = mock.Mock() self.job.scheduler.schedule_on_complete = False queued_job_run = mock.Mock() self.job.runs.get_first_queued = lambda: queued_job_run self.job_scheduler.handle_job_events(self.job, job.Job.NOTIFY_RUN_DONE) reactor.callLater.assert_any_call( 0, self.job_scheduler.run_job, queued_job_run, run_queued=True, ) def test_handle_job_events_schedule_on_complete(self): self.job_scheduler.schedule = mock.Mock() self.job.scheduler.schedule_on_complete = True self.job_scheduler.handle_job_events(self.job, job.Job.NOTIFY_RUN_DONE) self.job_scheduler.schedule.assert_called_with() def test_handler_unknown_event(self): self.job.runs.get_runs_by_state = mock.Mock() self.job_scheduler.handler(self.job, "some_other_event") self.job.runs.get_runs_by_state.assert_not_called() def test_handler_no_queued(self): self.job_scheduler.run_job = mock.Mock() def get_queued(state): if state == ActionRun.QUEUED: return [] self.job.runs.get_runs_by_state = get_queued self.job_scheduler.handler(self.job, job.Job.NOTIFY_RUN_DONE) self.job_scheduler.run_job.assert_not_called() @mock.patch("tron.core.job_scheduler.reactor", autospec=True) def test_run_queue_schedule(self, reactor): with mock.patch.object( self.job_scheduler, "schedule", ) as mock_schedule: self.job_scheduler.run_job = mock.Mock() self.job.scheduler.schedule_on_complete = False queued_job_run = mock.Mock() self.job.runs.get_first_queued = lambda: queued_job_run self.job_scheduler.run_queue_schedule() reactor.callLater.assert_called_once_with( 0, self.job_scheduler.run_job, queued_job_run, run_queued=True, ) mock_schedule.assert_called_once_with() class TestJobSchedulerOther(TestCase): """Test other JobScheduler functions""" def _make_job_scheduler(self, job_name, enabled=True): scheduler = mock.Mock() run_collection = mock.Mock() node_pool = mock.Mock() new_job = job.Job( job_name, scheduler, run_collection=run_collection, node_pool=node_pool, enabled=enabled, ) return new_job, JobScheduler(new_job) @setup def setup_job(self): self.job, self.job_scheduler = self._make_job_scheduler( "jobname", True, ) def test_disable(self): self.job.runs.cancel_pending = mock.Mock() self.job_scheduler.disable() assert not self.job.enabled assert self.job.runs.cancel_pending.call_count == 1 def test_update_from_job_scheduler_disable(self): new_job, new_job_scheduler = self._make_job_scheduler("jobname", False) self.job.update_from_job = mock.Mock() self.job_scheduler.disable = mock.Mock() self.job_scheduler.update_from_job_scheduler(new_job_scheduler) assert self.job.update_from_job.call_args == mock.call( new_job_scheduler.get_job(), ) assert self.job_scheduler.disable.call_count == 1 def test_update_from_job_scheduler_enable(self): new_job, new_job_scheduler = self._make_job_scheduler("jobname", True) self.job.update_from_job = mock.Mock() self.job.enabled = False self.job.config_enabled = False self.job_scheduler.enable = mock.Mock() self.job_scheduler.update_from_job_scheduler(new_job_scheduler) assert self.job.update_from_job.call_args == mock.call( new_job_scheduler.get_job(), ) assert self.job_scheduler.enable.call_count == 1 def test_update_from_job_scheduler_no_config_change(self): new_job, new_job_scheduler = self._make_job_scheduler("jobname", True) self.job.enabled = False self.job.update_from_job = mock.Mock() self.job_scheduler.enable = mock.Mock() self.job_scheduler.disable = mock.Mock() self.job_scheduler.update_from_job_scheduler(new_job_scheduler) assert self.job.update_from_job.call_args == mock.call( new_job_scheduler.get_job(), ) assert self.job_scheduler.enable.call_count == 0 assert self.job_scheduler.disable.call_count == 0 assert self.job.config_enabled == new_job.config_enabled assert not self.job.enabled class TestJobSchedulerFactory(TestCase): @setup def setup_factory(self): self.context = mock.Mock() self.output_stream_dir = mock.Mock() self.time_zone = mock.Mock() self.action_runner = mock.create_autospec( actioncommand.SubprocessActionRunnerFactory, ) self.factory = JobSchedulerFactory( self.context, self.output_stream_dir, self.time_zone, self.action_runner, mock.Mock(), ) def test_build(self): config = mock.Mock() with mock.patch( "tron.core.job_scheduler.Job", autospec=True, ) as mock_job: job_scheduler = self.factory.build(config) _, kwargs = mock_job.from_config.call_args assert_equal(kwargs["job_config"], config) assert_equal( job_scheduler.get_job(), mock_job.from_config.return_value, ) assert_equal(kwargs["parent_context"], self.context) assert_equal(kwargs["output_path"].base, self.output_stream_dir) assert_equal(kwargs["action_runner"], self.action_runner) ================================================ FILE: tests/core/job_test.py ================================================ import collections import datetime from unittest import mock from unittest.mock import MagicMock import pytest from testifycompat import assert_equal from testifycompat import assert_not_equal from tests.assertions import assert_call from tests.assertions import assert_length from tests.testingutils import autospec_method from tron import actioncommand from tron import node from tron.core import job from tron.core import jobrun from tron.core.actionrun import ActionRun from tron.core.job_scheduler import JobScheduler @pytest.fixture def mock_node_repo(): with mock.patch( "tron.core.job.node.NodePoolRepository", autospec=True, ) as mock_node_repo: yield mock_node_repo @pytest.fixture def mock_job(mock_node_repo): action_graph = mock.Mock(names=lambda: ["one", "two"]) scheduler = mock.Mock() run_collection = MagicMock() nodes = mock.create_autospec(node.NodePool) mock_job = job.Job( "jobname", scheduler, run_collection=run_collection, action_graph=action_graph, node_pool=nodes, action_runner=actioncommand.NoActionRunnerFactory, ) yield mock_job class TestJob: @pytest.fixture(autouse=True) def setup_job(self, mock_job): self.job = mock_job autospec_method(self.job.notify) autospec_method(self.job.watch) yield def test__init__(self): assert str(self.job.output_path).endswith(self.job.name) def test_from_config(self, mock_node_repo): action = mock.MagicMock( name="first", command="doit", node=None, requires=[], ) job_config = mock.Mock( node="thenodepool", monitoring={ "team": "foo", "page": True, }, all_nodes=False, queueing=True, enabled=True, run_limit=20, actions={action.name: action}, cleanup_action=None, ) job_config.name = "ajob" # set this after mock creation to give it a "real" name attribute scheduler = "scheduler_token" parent_context = "parent_context_token" output_path = ["base_path"] mock_action_runner = mock.create_autospec( actioncommand.SubprocessActionRunnerFactory, ) new_job = job.Job.from_config( job_config, scheduler, parent_context=parent_context, output_path=output_path, action_runner=mock_action_runner, action_graph=mock.Mock(), ) assert_equal(new_job.scheduler, scheduler) assert_equal(new_job.context.next, parent_context) mock_node_repo.get_instance().get_by_name.assert_called_with( job_config.node, ) assert_equal(new_job.enabled, True) assert_equal(new_job.get_monitoring()["team"], "foo") assert new_job.action_graph def test_update_from_job(self): action_runner = mock.Mock() other_job = job.Job( "otherjob", "scheduler", action_runner=action_runner, run_limit=10, ) self.job.update_from_job(other_job) assert_equal(self.job.name, "otherjob") assert_equal(self.job.scheduler, "scheduler") assert_equal(self.job, other_job) assert_equal(self.job.runs.run_limit, 10) def test_status_disabled(self): self.job.enabled = False assert_equal(self.job.status, self.job.STATUS_DISABLED) def test_status_enabled(self): self.job.runs.get_run_by_state = lambda state: MagicMock() if state == ActionRun.SCHEDULED else None self.job.runs.get_active.return_value = [] assert_equal(self.job.status, self.job.STATUS_ENABLED) def test_status_running(self): self.job.runs.get_active.return_value = [MagicMock()] assert_equal(self.job.status, self.job.STATUS_RUNNING) def test_status_unknown(self): self.job.runs.get_active.return_value = [] self.job.runs.get_run_by_state = lambda s: None assert_equal(self.job.status, self.job.STATUS_UNKNOWN) def test_state_data(self): state_data = self.job.state_data assert_equal(state_data["run_nums"], self.job.runs.get_run_nums.return_value) assert state_data["enabled"] def test_get_job_runs_from_state(self): job_runs = [ dict( run_num=i, job_name="thename", run_time="sometime", start_time="start_time", end_time="sometime", cleanup_run=None, runs=[], ) for i in range(0, 3) ] state_data = {"enabled": False, "runs": job_runs} self.job.get_job_runs_from_state(state_data) assert not self.job.enabled def test_build_new_runs(self): run_time = datetime.datetime(2012, 3, 14, 15, 9, 26) runs = list(self.job.build_new_runs(run_time)) self.job.node_pool.next.assert_called_with() node = self.job.node_pool.next.return_value assert_call( self.job.runs.build_new_run, 0, self.job, run_time, node, manual=False, ) assert_length(runs, 1) self.job.watch.assert_called_with(runs[0]) def test_build_new_runs_all_nodes(self): self.job.all_nodes = True run_time = datetime.datetime(2012, 3, 14, 15, 9, 26) node_count = 2 self.job.node_pool.nodes = [mock.Mock()] * node_count runs = list(self.job.build_new_runs(run_time)) assert_length(runs, node_count) for i in range(len(runs)): node = self.job.node_pool.nodes[i] assert_call( self.job.runs.build_new_run, i, self.job, run_time, node, manual=False, ) calls = [] for r in runs: calls.extend(r.mock_calls) self.job.watch.assert_has_calls(calls) def test_build_new_runs_manual(self): run_time = datetime.datetime(2012, 3, 14, 15, 9, 26) runs = list(self.job.build_new_runs(run_time, manual=True)) self.job.node_pool.next.assert_called_with() node = self.job.node_pool.next.return_value assert_length(runs, 1) assert_call( self.job.runs.build_new_run, 0, self.job, run_time, node, manual=True, ) self.job.watch.assert_called_with(runs[0]) def test_handler(self): self.job.handler(None, jobrun.JobRun.NOTIFY_STATE_CHANGED) self.job.notify.assert_called_with(self.job.NOTIFY_STATE_CHANGE) self.job.handler(None, jobrun.JobRun.NOTIFY_DONE) self.job.notify.assert_called_with(self.job.NOTIFY_RUN_DONE) def test__eq__(self): other_job = job.Job("jobname", "scheduler", run_collection=MagicMock()) assert not self.job == other_job other_job.update_from_job(self.job) assert_equal(self.job, other_job) def test__ne__(self): other_job = job.Job("jobname", "scheduler", run_collection=MagicMock()) assert self.job != other_job other_job.update_from_job(self.job) assert not self.job != other_job def test__eq__true(self): action_runner = mock.Mock() first = job.Job("jobname", "scheduler", action_runner=action_runner) second = job.Job("jobname", "scheduler", action_runner=action_runner) assert_equal(first, second) def test__eq__false(self): first = job.Job("jobname", "scheduler", action_runner=mock.Mock()) second = job.Job("jobname", "scheduler", action_runner=mock.Mock()) assert_not_equal(first, second) def test_job_watch_notifies_about_runs(mock_job): # Separate from the above tests because we don't want # watch to be mocked here. new_run = jobrun.JobRun( job_name="test", run_num=1, run_time="some_time", node="node", ) with mock.patch.object(mock_job, "handler",) as mock_handler, mock.patch.object( mock_job, "notify", ) as mock_notify: mock_job.watch(new_run) # Make sure that the job is still watching correctly # by checking it handles events new_run.notify("test_event", "test_data") assert mock_handler.call_args_list == [mock.call(new_run, "test_event", "test_data")] # Check that the job notifies its watchers about a new run assert mock_notify.call_args_list == [mock.call(job.Job.NOTIFY_NEW_RUN, event_data=new_run)] class TestJobScheduler: @pytest.fixture(autouse=True) def setup_job(self): mock_graph = mock.Mock(autospec=True) mock_graph.get_action_map.return_value = {} mock_graph.action_map = {} self.job = mock.Mock(autospec=True) self.job.allow_overlap = False self.job.max_runtime = datetime.timedelta(days=1) self.job_scheduler = JobScheduler(job=self.job) def test_restore_state_sets_job_runs(self): self.job.enabled = False mock_runs = [mock.Mock(), mock.Mock()] mock_action_runner = mock.Mock() job_state_data = {"runs": mock_runs, "enabled": True} self.job_scheduler._set_callback = lambda x: x self.job.runs.runs = collections.deque() self.job.runs.get_scheduled.return_value = [mock.Mock()] self.job.get_job_runs_from_state.return_value = mock_runs with mock.patch( "tron.core.job_scheduler.recovery.launch_recovery_actionruns_for_job_runs", autospec=True, ) as mock_launch_recovery: mock_launch_recovery.return_value = mock.Mock(autospec=True) self.job_scheduler.restore_state( job_state_data, mock_action_runner, ) assert self.job.runs.runs == collections.deque(mock_runs) mock_launch_recovery.assert_called_once_with( job_runs=mock_runs, master_action_runner=mock_action_runner, ) calls = [mock.call(mock_runs[i]) for i in range(0, len(mock_runs))] self.job.watch.assert_has_calls(calls) def test_create_and_schedule_runs_specific_time(self): self.job_scheduler.get_runs_to_schedule = mock.Mock(return_value=[mock.Mock()]) self.job_scheduler._set_callback = mock.Mock() self.job_scheduler.create_and_schedule_runs(next_run_time="a_datetime") assert self.job_scheduler.get_runs_to_schedule.call_args_list == [mock.call("a_datetime")] def test_create_and_schedule_runs_guess(self): self.job_scheduler.get_runs_to_schedule = mock.Mock(return_value=[mock.Mock()]) self.job_scheduler._set_callback = mock.Mock() self.job_scheduler.create_and_schedule_runs(next_run_time=None) assert self.job_scheduler.get_runs_to_schedule.call_args_list == [mock.call(None)] def test_disable(self): self.job_scheduler.disable() assert self.job_scheduler.job.enabled is False self.job_scheduler.job.runs.cancel_pending.assert_called_once() def test_schedule_reconfigured(self): pending_run = mock.Mock() pending_run.run_time = "a_run_time" self.job.runs.get_pending.return_value = [pending_run] self.job_scheduler.create_and_schedule_runs = mock.Mock() self.job_scheduler.schedule_reconfigured() assert self.job.runs.remove_pending.call_count == 1 assert self.job_scheduler.create_and_schedule_runs.call_args_list == [ mock.call( next_run_time="a_run_time", ), ] def test_schedule(self): self.job.enabled = True last_run = mock.Mock() last_run.run_time = "a_run_time" self.job.runs.get_newest = mock.Mock(return_value=last_run) self.job_scheduler.create_and_schedule_runs = mock.Mock() self.job_scheduler.schedule() self.job.scheduler.next_run_time.assert_called_once_with("a_run_time") assert self.job_scheduler.create_and_schedule_runs.call_args_list == [ mock.call(next_run_time=self.job.scheduler.next_run_time.return_value), ] def test_run_job(self): self.job_scheduler.schedule = mock.Mock(autospec=True) self.job.scheduler.schedule_on_complete = False self.job.runs.get_active = lambda n: [] job_run = mock.Mock(autospec=True) job_run.is_cancelled = False self.job_scheduler.run_job(job_run) job_run.start.assert_called_once() self.job_scheduler.schedule.assert_called_once() def test_run_job_job_disabled(self): self.job_scheduler.schedule = MagicMock() job_run = MagicMock() self.job.enabled = False self.job_scheduler.run_job(job_run) assert_length(self.job_scheduler.schedule.mock_calls, 0) assert_length(job_run.start.mock_calls, 0) assert_length(job_run.cancel.mock_calls, 1) def test_run_job_cancelled(self): self.job_scheduler.schedule = MagicMock() job_run = MagicMock(is_scheduled=False) self.job_scheduler.run_job(job_run) assert_length(job_run.start.mock_calls, 0) assert_length(self.job_scheduler.schedule.mock_calls, 1) def test_run_job_already_running_queuing(self): self.job_scheduler.schedule = mock.Mock(autospec=True) self.job.runs.get_active = lambda s: [mock.Mock(autospec=True)] job_run = mock.Mock(autospec=True) job_run.is_cancelled = False self.job_scheduler.run_job(job_run) assert not job_run.start.called job_run.queue.assert_called_once() assert not self.job_scheduler.schedule.called def test_run_job_already_running_cancel(self): self.job_scheduler.schedule = mock.Mock(autospec=True) self.job.runs.get_active = lambda s: [mock.Mock(autospec=True)] self.job.queueing = False job_run = mock.Mock(autospec=True) job_run.is_cancelled = False self.job_scheduler.run_job(job_run) assert not job_run.start.called job_run.cancel.assert_called_once() self.job_scheduler.schedule.assert_called_once() def test_run_job_already_running_allow_overlap(self): self.job_scheduler.schedule = mock.Mock() self.job.runs.get_active = lambda s: [mock.Mock()] self.job.allow_overlap = True job_run = MagicMock(is_cancelled=False) self.job_scheduler.run_job(job_run) job_run.start.assert_called_with() def test_run_job_has_starting_queueing(self): self.job_scheduler.schedule = mock.Mock(autospec=True) self.job.runs.get_active = lambda s: [mock.Mock(autospec=True)] job_run = mock.Mock(autospec=True) job_run.is_cancelled = False self.job_scheduler.run_job(job_run) assert not job_run.start.called job_run.queue.assert_called_once() assert not self.job_scheduler.schedule.called def test_run_job_schedule_on_complete(self): self.job_scheduler.schedule = MagicMock() self.job.scheduler.schedule_on_complete = True self.job.runs.get_active = lambda s: [] job_run = MagicMock(is_cancelled=False) self.job_scheduler.run_job(job_run) assert_length(job_run.start.mock_calls, 1) assert_length(self.job_scheduler.schedule.mock_calls, 0) ================================================ FILE: tests/core/jobgraph_test.py ================================================ from unittest import mock import pytest from tron.config.schema import ConfigAction from tron.config.schema import ConfigJob from tron.core.jobgraph import AdjListEntry from tron.core.jobgraph import JobGraph MISSING_DEPENDENCY_ERR_MSG = """The following actions are dependencies of other actions but missing: Action other.job2.action3 is dependency of actions: - MASTER.job3.action5 Please check if you have deleted/renamed any of them or their containing jobs.""" def _setup_job_graph_config_container(): action1 = ConfigAction( name="action1", command="do something", ) action2 = ConfigAction( name="action2", command="do something", requires=["action1"], ) job1_config = ConfigJob( name="job1", node="default", schedule=mock.Mock(), actions={"action1": action1, "action2": action2}, namespace="MASTER", ) action3 = ConfigAction( name="action3", command="do something", triggered_by=["MASTER.job1.action2.shortdate.{shortdate}"], ) job2_config = ConfigJob( name="job1", node="default", schedule=mock.Mock(), actions={"action3": action3}, namespace="other", ) action4 = ConfigAction( name="action4", command="do something", ) action5 = ConfigAction( name="action5", command="do something", requires=["action4"], triggered_by=["other.job2.action3.shortdate.{shortdate}"], ) job3_config = ConfigJob( name="job1", node="default", schedule=mock.Mock(), actions={"action4": action4, "action5": action5}, namespace="MASTER", ) config_container = mock.Mock() config_container.get_jobs.return_value = { "MASTER.job1": job1_config, "other.job2": job2_config, "MASTER.job3": job3_config, } return config_container class TestJobGraph: def setup_method(self): self.job_graph = JobGraph(_setup_job_graph_config_container(), should_validate_missing_dependency=True) def test_job_graph_missing_dependency(self): missing_dependency_config_container = _setup_job_graph_config_container() missing_dependency_config_container.get_jobs.return_value.pop("other.job2") with pytest.raises(ValueError) as e: JobGraph( missing_dependency_config_container, should_validate_missing_dependency=True, ) assert str(e.value) == MISSING_DEPENDENCY_ERR_MSG def test_job_graph(self): assert sorted(list(self.job_graph.action_map.keys())) == [ "MASTER.job1.action1", "MASTER.job1.action2", "MASTER.job3.action4", "MASTER.job3.action5", "other.job2.action3", ] assert self.job_graph._actions_for_job == { "MASTER.job1": ["MASTER.job1.action1", "MASTER.job1.action2"], "other.job2": ["other.job2.action3"], "MASTER.job3": ["MASTER.job3.action4", "MASTER.job3.action5"], } assert self.job_graph._adj_list == { "MASTER.job1.action1": [AdjListEntry("MASTER.job1.action2", False)], "MASTER.job1.action2": [AdjListEntry("other.job2.action3", True)], "other.job2.action3": [AdjListEntry("MASTER.job3.action5", True)], "MASTER.job3.action4": [AdjListEntry("MASTER.job3.action5", False)], } assert self.job_graph._rev_adj_list == { "MASTER.job1.action1": [], "MASTER.job1.action2": [AdjListEntry("MASTER.job1.action1", False)], "other.job2.action3": [AdjListEntry("MASTER.job1.action2", True)], "MASTER.job3.action4": [], "MASTER.job3.action5": [ AdjListEntry("MASTER.job3.action4", False), AdjListEntry("other.job2.action3", True), ], } def test_get_action_graph_for_job(self): action_graph_1 = self.job_graph.get_action_graph_for_job("MASTER.job1") assert sorted(action_graph_1.action_map.keys()) == [ "action1", "action2", ] assert action_graph_1.required_actions == { "action1": set(), "action2": {"action1"}, } assert action_graph_1.required_triggers == { "other.job2.action3": {"action2"}, "MASTER.job3.action5": {"other.job2.action3"}, } action_graph_2 = self.job_graph.get_action_graph_for_job("other.job2") assert sorted(action_graph_2.action_map.keys()) == [ "action3", ] assert action_graph_2.required_actions == { "action3": set(), } assert action_graph_2.required_triggers == { "action3": {"MASTER.job1.action2"}, "MASTER.job3.action5": {"action3"}, } action_graph_3 = self.job_graph.get_action_graph_for_job("MASTER.job3") assert sorted(action_graph_3.action_map.keys()) == [ "action4", "action5", ] assert action_graph_3.required_actions == { "action4": set(), "action5": {"action4"}, } assert action_graph_3.required_triggers == { "action5": {"other.job2.action3"}, "other.job2.action3": {"MASTER.job1.action2"}, } ================================================ FILE: tests/core/jobrun_test.py ================================================ import datetime import json from unittest import mock from unittest.mock import MagicMock import pytest import pytz from testifycompat import assert_equal from testifycompat import assert_in from testifycompat import setup from testifycompat import TestCase from tests.assertions import assert_call from tests.assertions import assert_length from tests.assertions import assert_raises from tests.testingutils import autospec_method from tron import actioncommand from tron import node from tron.core import action from tron.core import actiongraph from tron.core import actionrun from tron.core import job from tron.core import jobrun from tron.serialize import filehandler def build_mock_job(): action_graph = mock.create_autospec(actiongraph.ActionGraph) action_graph.action_map = { "foo": mock.Mock( triggered_by=[], trigger_timeout=datetime.timedelta(days=1), ), } runner = mock.create_autospec(actioncommand.SubprocessActionRunnerFactory) return mock.create_autospec( job.Job, action_graph=action_graph, output_path=mock.Mock(), context=mock.Mock(), action_runner=runner, ) class TestJobRun: now = datetime.datetime(2012, 3, 14, 15, 9, 20, tzinfo=None) now_with_tz = datetime.datetime(2012, 3, 14, 15, 9, 20, tzinfo=pytz.utc) @setup def setup_jobrun(self): self.job = build_mock_job() self.action_graph = self.job.action_graph self.run_time = datetime.datetime(2012, 3, 14, 15, 9, 26) mock_node = mock.create_autospec(node.Node) self.job_run = jobrun.JobRun( "jobname", 7, self.run_time, mock_node, action_runs=MagicMock( action_runs_with_cleanup=[], get_startable_action_runs=lambda: [], ), ) autospec_method(self.job_run.watch) autospec_method(self.job_run.notify) self.action_run = mock.create_autospec( actionrun.ActionRun, is_skipped=False, ) def test__init__(self): assert_equal(self.job_run.job_name, "jobname") assert_equal(self.job_run.run_time, self.run_time) assert str(self.job_run.output_path).endswith(str(self.job_run.run_num)) def test_for_job(self): run_num = 6 mock_node = mock.create_autospec(node.Node) run = jobrun.JobRun.for_job( self.job, run_num, self.run_time, mock_node, False, ) assert_equal(run.action_runs.action_graph, self.action_graph) assert_equal(run.job_name, self.job.get_name.return_value) assert_equal(run.run_num, run_num) assert_equal(run.node, mock_node) assert not run.manual def test_for_job_manual(self): run_num = 6 mock_node = mock.create_autospec(node.Node) run = jobrun.JobRun.for_job( self.job, run_num, self.run_time, mock_node, True, ) assert_equal(run.action_runs.action_graph, self.action_graph) assert run.manual def test_state_data(self): state_data = self.job_run.state_data assert_equal(state_data["run_num"], 7) assert not state_data["manual"] assert_equal(state_data["run_time"], self.run_time) def test_set_action_runs(self): self.job_run._action_runs = None count = 2 action_runs = [mock.create_autospec(actionrun.ActionRun) for _ in range(count)] run_collection = mock.create_autospec( actionrun.ActionRunCollection, action_runs_with_cleanup=action_runs, ) self.job_run._set_action_runs(run_collection) assert_equal(self.job_run.watch.call_count, count) expected = [mock.call(run) for run in action_runs] assert_equal(self.job_run.watch.mock_calls, expected) assert_equal(self.job_run.action_runs, run_collection) assert self.job_run.action_runs_proxy def test_set_action_runs_none(self): self.job_run._action_runs = None run_collection = mock.create_autospec(actionrun.ActionRunCollection) self.job_run._set_action_runs(run_collection) assert not self.job_run.watch.mock_calls assert_equal(self.job_run.action_runs, run_collection) def test_set_action_runs_duplicate(self): run_collection = mock.create_autospec(actionrun.ActionRunCollection) assert_raises( ValueError, self.job_run._set_action_runs, run_collection, ) @mock.patch("tron.core.jobrun.timeutils.current_time", autospec=True) def test_seconds_until_run_time(self, mock_current_time): mock_current_time.return_value = self.now seconds = self.job_run.seconds_until_run_time() assert_equal(seconds, 6) @mock.patch("tron.core.jobrun.timeutils.current_time", autospec=True) def test_seconds_until_run_time_with_tz(self, mock_current_time): mock_current_time.return_value = self.now_with_tz self.job_run.run_time = self.run_time.replace(tzinfo=pytz.utc) seconds = self.job_run.seconds_until_run_time() assert_equal(seconds, 6) def test_start(self): autospec_method(self.job_run._do_start) assert self.job_run.start() self.job_run._do_start.assert_called_with() def test_start_failed(self): autospec_method(self.job_run._do_start, return_value=False) assert not self.job_run.start() def test_do_start(self): startable_runs = [mock.create_autospec(actionrun.ActionRun) for _ in range(3)] self.job_run.action_runs.get_startable_action_runs = lambda: startable_runs assert self.job_run._do_start() self.job_run.action_runs.ready.assert_called_with() for startable_run in startable_runs: startable_run.start.assert_called_with() def test_do_start_all_failed(self): autospec_method(self.job_run._start_action_runs, return_value=[None]) assert not self.job_run._do_start() def test_do_start_some_failed(self): returns = [True, None] autospec_method(self.job_run._start_action_runs, return_value=returns) assert self.job_run._do_start() def test_do_start_no_runs(self): assert not self.job_run._do_start() def test_start_action_runs(self): startable_runs = [mock.create_autospec(actionrun.ActionRun) for _ in range(3)] self.job_run.action_runs.get_startable_action_runs = lambda: startable_runs started_runs = self.job_run._start_action_runs() assert_equal(started_runs, startable_runs) def test_start_action_runs_failed(self): startable_runs = [mock.create_autospec(actionrun.ActionRun) for _ in range(3)] startable_runs[0].start.return_value = False self.job_run.action_runs.get_startable_action_runs = lambda: startable_runs started_runs = self.job_run._start_action_runs() assert_equal(started_runs, startable_runs[1:]) @pytest.fixture def jobrun_json(self): runs = [ { "job_run_id": "compute-infra-test-service.test_load_foo1.5910", "action_name": "example_action", "state": "succeeded", "original_command": "date; sleep 150; date", "start_time": "2023-10-01T12:00:00", "end_time": "2023-10-01T12:30:00", "node_name": "paasta", "exit_status": 0, "attempts": [], "retries_remaining": 2, "retries_delay": 60, "action_runner": '{"status_path": "/tmp/tron", "exec_path": "/opt/venvs/tron/bin"}', "executor": "kubernetes", "trigger_timeout_timestamp": 1731584100, "trigger_downstreams": False, "triggered_by": [], "on_upstream_rerun": None, } ] cleanup = { "job_run_id": "compute-infra-test-service.test_load_foo1.5910", "action_name": "cleanup_action", "state": "succeeded", "original_command": "date; sleep 150; date", "start_time": "2023-10-01T12:00:00", "end_time": "2023-10-01T12:30:00", "node_name": "paasta", "exit_status": 0, "attempts": [], "retries_remaining": 2, "retries_delay": 60, "action_runner": '{"status_path": "/tmp/tron", "exec_path": "/opt/venvs/tron/bin"}', "executor": "kubernetes", "trigger_timeout_timestamp": 1731584100, "trigger_downstreams": False, "triggered_by": [], "on_upstream_rerun": None, } serialized_cleanup = json.dumps(cleanup) serialized_runs = [json.dumps(run) for run in runs] return json.dumps( { "job_name": "example_job", "run_num": 1, "run_time": "2023-10-01T12:00:00", "time_zone": None, "node_name": "example_node", "runs": serialized_runs, "cleanup_run": serialized_cleanup, "manual": False, } ) def test_from_json(self, jobrun_json): result = jobrun.JobRun.from_json(jobrun_json) expected = { "job_name": "example_job", "run_num": 1, "run_time": datetime.datetime(2023, 10, 1, 12, 0, 0), "node_name": "example_node", "runs": [ { "job_run_id": "compute-infra-test-service.test_load_foo1.5910", "action_name": "example_action", "state": "succeeded", "original_command": "date; sleep 150; date", "start_time": datetime.datetime(2023, 10, 1, 12, 0, 0), "end_time": datetime.datetime(2023, 10, 1, 12, 30, 0), "node_name": "paasta", "exit_status": 0, "attempts": [], "retries_remaining": 2, "retries_delay": datetime.timedelta(seconds=60), "action_runner": {"status_path": "/tmp/tron", "exec_path": "/opt/venvs/tron/bin"}, "executor": "kubernetes", "trigger_timeout_timestamp": 1731584100, "trigger_downstreams": False, "triggered_by": [], "on_upstream_rerun": None, } ], "cleanup_run": { "job_run_id": "compute-infra-test-service.test_load_foo1.5910", "action_name": "cleanup_action", "state": "succeeded", "original_command": "date; sleep 150; date", "start_time": datetime.datetime(2023, 10, 1, 12, 0, 0), "end_time": datetime.datetime(2023, 10, 1, 12, 30, 0), "node_name": "paasta", "exit_status": 0, "attempts": [], "retries_remaining": 2, "retries_delay": datetime.timedelta(seconds=60), "action_runner": {"status_path": "/tmp/tron", "exec_path": "/opt/venvs/tron/bin"}, "executor": "kubernetes", "trigger_timeout_timestamp": 1731584100, "trigger_downstreams": False, "triggered_by": [], "on_upstream_rerun": None, }, "manual": False, "time_zone": None, } assert result == expected def test_start_action_runs_all_failed(self): startable_runs = [mock.create_autospec(actionrun.ActionRun) for _ in range(2)] for startable_run in startable_runs: startable_run.start.return_value = False self.job_run.action_runs.get_startable_action_runs = lambda: startable_runs started_runs = self.job_run._start_action_runs() assert_equal(started_runs, []) def test_handler_trigger_ready_still_scheduled(self): autospec_method(self.job_run._start_action_runs) self.job_run.is_scheduled = True self.job_run.handler(self.action_run, actionrun.ActionRun.NOTIFY_TRIGGER_READY) assert not self.job_run._start_action_runs.mock_calls def test_handler_trigger_ready_started(self): autospec_method(self.job_run._start_action_runs) self.job_run.is_scheduled = False self.job_run.is_queued = False self.job_run.handler(self.action_run, actionrun.ActionRun.NOTIFY_TRIGGER_READY) assert self.job_run._start_action_runs.call_count == 1 def test_handler_not_end_state_event(self): autospec_method(self.job_run.finalize) autospec_method(self.job_run._start_action_runs) self.action_run.is_done = False self.job_run.handler(self.action_run, mock.Mock()) assert not self.job_run.finalize.mock_calls assert not self.job_run._start_action_runs.mock_calls def test_handler_with_startable(self): startable_run = mock.create_autospec(actionrun.ActionRun) self.job_run.action_runs.get_startable_action_runs = lambda: [ startable_run, ] autospec_method(self.job_run.finalize) self.action_run.is_broken = False self.job_run.handler(self.action_run, mock.Mock()) self.job_run.notify.assert_called_with( self.job_run.NOTIFY_STATE_CHANGED, ) startable_run.start.assert_called_with() assert not self.job_run.finalize.mock_calls def test_handler_runs_not_done(self): self.job_run.action_runs.is_done = False autospec_method(self.job_run._start_action_runs, return_value=[]) autospec_method(self.job_run.finalize) self.job_run.handler(self.action_run, mock.Mock()) assert not self.job_run.finalize.mock_calls def test_handler_finished_without_cleanup(self): self.job_run.action_runs.is_active = False self.job_run.action_runs.is_scheduled = False self.job_run.action_runs.cleanup_action_run = None autospec_method(self.job_run.finalize) self.job_run.handler(self.action_run, mock.Mock()) self.job_run.finalize.assert_called_with() def test_handler_finished_with_cleanup_done(self): self.job_run.action_runs.is_active = False self.job_run.action_runs.is_scheduled = False self.job_run.action_runs.cleanup_action_run = mock.Mock(is_done=True) autospec_method(self.job_run.finalize) self.job_run.handler(self.action_run, mock.Mock()) self.job_run.finalize.assert_called_with() def test_handler_finished_with_cleanup(self): self.job_run.action_runs.is_active = False self.job_run.action_runs.is_scheduled = False self.job_run.action_runs.cleanup_action_run = mock.Mock(is_done=False) autospec_method(self.job_run.finalize) self.job_run.handler(self.action_run, mock.Mock()) assert not self.job_run.finalize.mock_calls self.job_run.action_runs.cleanup_action_run.start.assert_called_with() def test_handler_action_run_cancelled(self): self.action_run.is_broken = True autospec_method(self.job_run._start_action_runs) self.job_run.handler(self.action_run, mock.Mock()) assert not self.job_run._start_action_runs.mock_calls def test_handler_action_run_skipped(self): self.action_run.is_broken = False self.action_run.is_skipped = True self.job_run.action_runs.is_scheduled = True autospec_method(self.job_run._start_action_runs) self.job_run.handler(self.action_run, mock.Mock()) assert not self.job_run._start_action_runs.mock_calls def test_state(self): assert_equal(self.job_run.state, actionrun.ActionRun.SUCCEEDED) def test_state_with_no_action_runs(self): self.job_run._action_runs = None assert_equal(self.job_run.state, actionrun.ActionRun.UNKNOWN) def test_finalize(self): self.job_run.action_runs.is_failed = False self.job_run.finalize() self.job_run.notify.assert_called_with(self.job_run.NOTIFY_DONE) def test_finalize_failure(self): self.job_run.finalize() self.job_run.notify.assert_called_with(self.job_run.NOTIFY_DONE) def test_cleanup(self): autospec_method(self.job_run.clear_observers) self.job_run.output_path = mock.create_autospec(filehandler.OutputPath) self.job_run.cleanup() self.job_run.notify.assert_called_with(jobrun.JobRun.NOTIFY_REMOVED) self.job_run.clear_observers.assert_called_with() self.job_run.output_path.delete.assert_called_with() assert not self.job_run.node assert not self.job_run.action_graph assert not self.job_run.action_runs def test__getattr__(self): assert self.job_run.cancel assert self.job_run.state == "succeeded" assert self.job_run.is_succeeded def test__getattr__miss(self): assert_raises(AttributeError, lambda: self.job_run.bogus) class TestJobRunFromState(TestCase): @setup def setup_jobrun(self): self.action_graph = mock.create_autospec(actiongraph.ActionGraph, action_map={}) self.run_time = datetime.datetime(2012, 3, 14, 15, 9, 26) self.path = ["base", "path"] self.output_path = mock.create_autospec(filehandler.OutputPath) self.node_pool = mock.create_autospec(node.NodePool) self.action_run_state_data = [ { "job_run_id": "thejobname.22", "action_name": "blingaction", "state": "succeeded", "run_time": "sometime", "start_time": "sometime", "end_time": "sometime", "command": "doit", "node_name": "thenode", } ] self.state_data = { "job_name": "thejobname", "run_num": 22, "run_time": self.run_time, "node_name": "thebox", "end_time": "the_end", "start_time": "start_time", "runs": self.action_run_state_data, "cleanup_run": None, "manual": True, } self.context = mock.Mock() def test_from_state(self): run = jobrun.JobRun.from_state( self.state_data, self.action_graph, self.output_path, self.context, self.node_pool, ) assert_length(run.action_runs.run_map, 1) assert_equal(run.job_name, self.state_data["job_name"]) assert_equal(run.run_time, self.run_time) assert run.manual assert_equal(run.output_path, self.output_path) assert run.context.next assert run.action_graph def test_from_state_node_no_longer_exists(self): run = jobrun.JobRun.from_state( self.state_data, self.action_graph, self.output_path, self.context, self.node_pool, ) assert_length(run.action_runs.run_map, 1) assert_equal(run.job_name, "thejobname") assert_equal(run.run_time, self.run_time) assert_equal(run.node, self.node_pool) class MockJobRun(MagicMock): manual = False node = "anode" @property def is_scheduled(self): return self.state == actionrun.ActionRun.SCHEDULED @property def is_queued(self): return self.state == actionrun.ActionRun.QUEUED @property def is_running(self): return self.state == actionrun.ActionRun.RUNNING @property def is_starting(self): return self.state == actionrun.ActionRun.STARTING @property def is_waiting(self): return self.state == actionrun.ActionRun.WAITING def __repr__(self): return str(self.__dict__) class TestJobRunCollection(TestCase): def _mock_run(self, **kwargs): return MockJobRun(**kwargs) @setup def setup_runs(self): self.run_collection = jobrun.JobRunCollection(6) self.job_runs = [ self._mock_run(state=actionrun.ActionRun.QUEUED, run_num=5), self._mock_run(state=actionrun.ActionRun.WAITING, run_num=4), self._mock_run(state=actionrun.ActionRun.RUNNING, run_num=3), ] + [ self._mock_run( state=actionrun.ActionRun.SUCCEEDED, run_num=i, ) for i in range(2, 0, -1) ] self.run_collection.runs.extend(self.job_runs) self.mock_node = mock.create_autospec(node.Node) def test__init__(self): assert_equal(self.run_collection.run_limit, 6) def test_from_config(self): job_config = mock.Mock(run_limit=20) runs = jobrun.JobRunCollection.from_config(job_config) assert_equal(runs.run_limit, 20) def test_job_runs_from_state(self): state_data = [ dict( run_num=i, job_name="thename", run_time="sometime", start_time="start_time", end_time="sometime", cleanup_run=None, runs=[], ) for i in range(3, -1, -1) ] action_graph = mock.create_autospec(actiongraph.ActionGraph) output_path = mock.create_autospec(filehandler.OutputPath) context = mock.Mock() node_pool = mock.create_autospec(node.NodePool) runs = jobrun.job_runs_from_state( state_data, action_graph, output_path, context, node_pool, ) assert len(runs) == 4 assert all([type(job) == jobrun.JobRun for job in runs]) def test_build_new_run(self): autospec_method(self.run_collection.remove_old_runs) run_time = datetime.datetime(2012, 3, 14, 15, 9, 26) mock_job = build_mock_job() job_run = self.run_collection.build_new_run( mock_job, run_time, self.mock_node, ) assert_in(job_run, self.run_collection.runs) self.run_collection.remove_old_runs.assert_called_with() assert job_run.run_num == 6 assert job_run.job_name == mock_job.get_name.return_value def test_build_new_run_manual(self): autospec_method(self.run_collection.remove_old_runs) run_time = datetime.datetime(2012, 3, 14, 15, 9, 26) mock_job = build_mock_job() job_run = self.run_collection.build_new_run( mock_job, run_time, self.mock_node, True, ) assert_in(job_run, self.run_collection.runs) self.run_collection.remove_old_runs.assert_called_with() assert job_run.run_num == 6 assert job_run.manual def test_cancel_pending(self): pending_runs = [mock.Mock() for _ in range(2)] autospec_method( self.run_collection.get_pending, return_value=pending_runs, ) self.run_collection.cancel_pending() for pending_run in pending_runs: pending_run.cancel.assert_called_with() def test_cancel_pending_no_pending(self): autospec_method(self.run_collection.get_pending, return_value=[]) self.run_collection.cancel_pending() def test_remove_pending(self): self.run_collection.remove_pending() assert_length(self.run_collection.runs, 4) assert_equal(self.run_collection.runs[0], self.job_runs[1]) assert_call(self.job_runs[0].cleanup, 0) def test_get_run_by_state(self): state = actionrun.ActionRun.SUCCEEDED run = self.run_collection.get_run_by_state(state) assert_equal(run, self.job_runs[3]) def test_get_run_by_state_no_match(self): state = actionrun.ActionRun.UNKNOWN run = self.run_collection.get_run_by_state(state) assert_equal(run, None) def test_get_run_by_num(self): run = self.run_collection.get_run_by_num(1) assert_equal(run.run_num, 1) def test_get_run_by_num_no_match(self): run = self.run_collection.get_run_by_num(7) assert_equal(run, None) def test_get_run_by_index(self): run = self.run_collection.get_run_by_index(-1) assert_equal(run, self.job_runs[0]) run = self.run_collection.get_run_by_index(-2) assert_equal(run, self.job_runs[1]) run = self.run_collection.get_run_by_index(0) assert_equal(run, self.job_runs[-1]) run = self.run_collection.get_run_by_index(1) assert_equal(run, self.job_runs[-2]) def test_get_run_by_index_invalid_index(self): run = self.run_collection.get_run_by_index(-6) assert_equal(run, None) run = self.run_collection.get_run_by_index(5) assert_equal(run, None) def test_get_newest(self): run = self.run_collection.get_newest() assert_equal(run, self.job_runs[0]) def test_get_newest_exclude_manual(self): run = self._mock_run( state=actionrun.ActionRun.RUNNING, run_num=5, manual=True, ) self.job_runs.insert(0, run) newest_run = self.run_collection.get_newest(include_manual=False) assert_equal(newest_run, self.job_runs[1]) def test_get_newest_no_runs(self): run_collection = jobrun.JobRunCollection(5) assert_equal(run_collection.get_newest(), None) def test_pending(self): run_num = self.run_collection.next_run_num() scheduled_run = self._mock_run( run_num=run_num, state=actionrun.ActionRun.SCHEDULED, ) self.run_collection.runs.appendleft(scheduled_run) pending = list(self.run_collection.get_pending()) assert_length(pending, 2) assert_equal(pending, [scheduled_run, self.job_runs[0]]) def test_get_active(self): starting_run = self._mock_run( run_num=self.run_collection.next_run_num(), state=actionrun.ActionRun.STARTING, ) self.run_collection.runs.appendleft(starting_run) active = list(self.run_collection.get_active()) assert_length(active, 3) assert_equal(active, [starting_run, self.job_runs[1], self.job_runs[2]]) def test_get_active_with_node(self): starting_run = self._mock_run( run_num=self.run_collection.next_run_num(), state=actionrun.ActionRun.STARTING, ) starting_run.node = "differentnode" self.run_collection.runs.appendleft(starting_run) active = list(self.run_collection.get_active("anode")) assert_length(active, 2) assert_equal(active, [self.job_runs[1], self.job_runs[2]]) def test_get_active_none(self): active = list(self.run_collection.get_active("bogus")) assert_length(active, 0) def test_get_first_queued(self): run_num = self.run_collection.next_run_num() second_queued = self._mock_run( run_num=run_num, state=actionrun.ActionRun.QUEUED, ) self.run_collection.runs.appendleft(second_queued) first_queued = self.run_collection.get_first_queued() assert_equal(first_queued, self.job_runs[0]) def test_get_first_queued_no_match(self): self.job_runs[0].state = actionrun.ActionRun.CANCELLED first_queued = self.run_collection.get_first_queued() assert not first_queued def test_get_next_run_num(self): assert_equal(self.run_collection.next_run_num(), 6) def test_get_next_run_num_first(self): run_collection = jobrun.JobRunCollection(5) assert_equal(run_collection.next_run_num(), 0) def test_remove_old_runs(self): self.run_collection.run_limit = 1 self.run_collection.remove_old_runs() assert_length(self.run_collection.runs, 1) assert_call(self.job_runs[-1].cleanup, 0) for job_run in self.run_collection.runs: assert_length(job_run.cancel.calls, 0) def test_remove_old_runs_none(self): self.run_collection.remove_old_runs() for job_run in self.job_runs: assert_length(job_run.cancel.calls, 0) def test_remove_old_runs_no_runs(self): run_collection = jobrun.JobRunCollection(4) run_collection.remove_old_runs() def test_state_data(self): assert_length(self.run_collection.state_data, len(self.job_runs)) def test_last_success(self): assert_equal(self.run_collection.last_success, self.job_runs[3]) def test__str__(self): expected = "JobRunCollection[5(queued), 4(waiting), 3(running), 2(succeeded), 1(succeeded)]" assert_equal(str(self.run_collection), expected) def test_get_action_runs(self): action_name = "action_name" self.run_collection.runs = job_runs = [mock.Mock(), mock.Mock()] runs = self.run_collection.get_action_runs(action_name) expected = [job_run.get_action_run.return_value for job_run in job_runs] assert_equal(runs, expected) for job_run in job_runs: job_run.get_action_run.assert_called_with(action_name) def test_get_run_nums(self): assert self.run_collection.get_run_nums() == [5, 4, 3, 2, 1] class TestJobRunStateTransitions: """Integration test for the state of a job run when actions change state in various ways.""" @pytest.fixture def mock_event_bus(self): with mock.patch( "tron.core.actionrun.EventBus", autospec=True, ) as mock_event_bus: mock_event_bus.has_event.return_value = True yield mock_event_bus @pytest.fixture def job_run(self, tmpdir, mock_event_bus): action_foo = action.Action("foo", action.ActionCommandConfig("command"), None) action_after_foo = action.Action("after_foo", action.ActionCommandConfig("command"), None) action_bar = action.Action("bar", action.ActionCommandConfig("command"), None, triggered_by={"trigger"}) action_graph = actiongraph.ActionGraph( action_map={ "foo": action_foo, "after_foo": action_after_foo, "bar": action_bar, }, required_actions={"foo": set(), "after_foo": {"foo"}, "bar": set()}, required_triggers={"foo": set(), "after_foo": set(), "bar": {"trigger"}}, ) mock_job = mock.Mock( output_path=filehandler.OutputPath(tmpdir), action_graph=action_graph, action_runner=actioncommand.NoActionRunnerFactory(), ) job_run = jobrun.JobRun.for_job( mock_job, run_num=1, run_time=datetime.datetime.now(), node=mock.Mock(), manual=False, ) return job_run def test_success_path(self, job_run): # Check expected states as actions run normally and succeed. foo = job_run.get_action_run("foo") after_foo = job_run.get_action_run("after_foo") bar = job_run.get_action_run("bar") # Run is initially SCHEDULED assert job_run.state == actionrun.ActionRun.SCHEDULED # After starting, both actions without dependencies start. # Run is STARTING job_run.start() assert foo.is_starting assert bar.is_starting assert job_run.state == actionrun.ActionRun.STARTING # Commands start successfully, run is RUNNING. foo.action_command.started() bar.action_command.started() assert job_run.state == actionrun.ActionRun.RUNNING # Still RUNNING after one of two running actions succeeds bar.action_command.exited(0) assert job_run.state == actionrun.ActionRun.RUNNING # after_foo starts after its dependency succeeds foo.action_command.exited(0) assert after_foo.is_starting after_foo.action_command.started() assert job_run.state == actionrun.ActionRun.RUNNING # SUCCEEDED after all actions succeed after_foo.action_command.exited(0) assert job_run.state == actionrun.ActionRun.SUCCEEDED def test_one_action_fails(self, job_run): foo = job_run.get_action_run("foo") after_foo = job_run.get_action_run("after_foo") bar = job_run.get_action_run("bar") # bar action fails, job is RUNNING because foo is still running job_run.start() foo.action_command.started() bar.action_command.started() bar.action_command.exited(1) assert job_run.state == actionrun.ActionRun.RUNNING # after_foo still starts after its dependency succeeds foo.action_command.exited(0) assert after_foo.is_starting after_foo.action_command.started() assert job_run.state == actionrun.ActionRun.RUNNING # After running actions finish, run enters FAILED terminal state after_foo.action_command.exited(0) assert job_run.state == actionrun.ActionRun.FAILED # If we skip the failed action, run becomes SUCCEEDED bar.skip() assert job_run.state == actionrun.ActionRun.SUCCEEDED def test_one_action_unknown(self, job_run): foo = job_run.get_action_run("foo") after_foo = job_run.get_action_run("after_foo") bar = job_run.get_action_run("bar") assert job_run.state == actionrun.ActionRun.SCHEDULED # bar action becomes unknown, job is RUNNING because foo is still running job_run.start() foo.action_command.started() bar.action_command.started() bar.action_command.exited(None) assert job_run.state == actionrun.ActionRun.RUNNING # after_foo still starts after its dependency succeeds foo.action_command.exited(0) assert after_foo.is_starting after_foo.action_command.started() assert job_run.state == actionrun.ActionRun.RUNNING # UNKNOWN after running actions finish after_foo.action_command.exited(0) assert job_run.state == actionrun.ActionRun.UNKNOWN def test_both_unknown_and_failed(self, job_run): foo = job_run.get_action_run("foo") after_foo = job_run.get_action_run("after_foo") bar = job_run.get_action_run("bar") # bar action becomes unknown, job is RUNNING because foo is still running job_run.start() foo.action_command.started() bar.action_command.started() bar.action_command.exited(None) assert job_run.state == actionrun.ActionRun.RUNNING # after_foo still starts after its dependency succeeds foo.action_command.exited(0) assert after_foo.is_starting after_foo.action_command.started() assert job_run.state == actionrun.ActionRun.RUNNING # A different action fails # Overall run is FAILED after_foo.action_command.exited(1) assert job_run.state == actionrun.ActionRun.FAILED def test_required_action_fails(self, job_run): foo = job_run.get_action_run("foo") after_foo = job_run.get_action_run("after_foo") bar = job_run.get_action_run("bar") assert job_run.state == actionrun.ActionRun.SCHEDULED # An action (foo) required by another action fails # Run is RUNNING while the other action, bar, is running job_run.start() foo.action_command.started() bar.action_command.started() foo.action_command.exited(1) assert job_run.state == actionrun.ActionRun.RUNNING # bar action succeeds # after_foo cannot run because its required action failed # So run is FAILED even though after_foo is waiting bar.action_command.exited(0) assert after_foo.is_waiting assert job_run.state == actionrun.ActionRun.FAILED # Pretend we reconfigured and after_foo doesn't depend on foo anymore # Run should not be WAITING # Ideally it would still be FAILED, but for now it's UNKNOWN in this case job_run.action_runs.action_graph.required_actions["after_foo"] = {} assert job_run.state == actionrun.ActionRun.UNKNOWN def test_required_action_unknown(self, job_run): foo = job_run.get_action_run("foo") after_foo = job_run.get_action_run("after_foo") bar = job_run.get_action_run("bar") # An action (foo) required by another action becomes unknown # Run is RUNNING while the other action, bar, is running job_run.start() foo.action_command.started() bar.action_command.started() foo.action_command.exited(None) assert job_run.state == actionrun.ActionRun.RUNNING # Other action succeeds # after_foo cannot run because its required action is unknown # So run is UNKNOWN even though after_foo is waiting bar.action_command.exited(0) assert after_foo.is_waiting assert job_run.state == actionrun.ActionRun.UNKNOWN # Pretend we reconfigured and after_foo doesn't depend on foo anymore # Run should not be waiting job_run.action_runs.action_graph.required_actions["after_foo"] = {} assert job_run.state == actionrun.ActionRun.UNKNOWN def test_with_trigger(self, job_run, mock_event_bus): foo = job_run.get_action_run("foo") after_foo = job_run.get_action_run("after_foo") bar = job_run.get_action_run("bar") # Start without trigger for bar mock_event_bus.has_event.return_value = False # Job should still start in scheduled state assert job_run.state == actionrun.ActionRun.SCHEDULED # Only foo is able to start job_run.start() assert foo.is_starting assert bar.is_waiting assert job_run.state == actionrun.ActionRun.STARTING foo.action_command.started() assert job_run.state == actionrun.ActionRun.RUNNING # after_foo runs normally after foo succeeds foo.action_command.exited(0) assert after_foo.is_starting after_foo.action_command.started() assert job_run.state == actionrun.ActionRun.RUNNING # After after_foo succeeds, run is not done # WAITING because bar is still waiting for a trigger after_foo.action_command.exited(0) assert job_run.state == actionrun.ActionRun.WAITING # After trigger is available, job run finishes as normal mock_event_bus.has_event.return_value = True bar.trigger_notify() assert bar.is_starting bar.action_command.started() bar.action_command.exited(0) assert job_run.state == actionrun.ActionRun.SUCCEEDED def test_queued(self, job_run): assert job_run.state == actionrun.ActionRun.SCHEDULED job_run.queue() assert job_run.state == actionrun.ActionRun.QUEUED job_run.start() assert job_run.state == actionrun.ActionRun.STARTING def test_cancel_one(self, job_run): assert job_run.state == actionrun.ActionRun.SCHEDULED job_run.start() assert job_run.state == actionrun.ActionRun.STARTING job_run.get_action_run("after_foo").cancel() assert job_run.state == actionrun.ActionRun.CANCELLED ================================================ FILE: tests/core/recovery_test.py ================================================ from unittest import mock from unittest.mock import Mock from testifycompat import setup from testifycompat import TestCase from tron.actioncommand import NoActionRunnerFactory from tron.actioncommand import SubprocessActionRunnerFactory from tron.core.actionrun import ActionRun from tron.core.actionrun import KubernetesActionRun from tron.core.actionrun import MesosActionRun from tron.core.actionrun import SSHActionRun from tron.core.recovery import filter_action_runs_needing_recovery from tron.core.recovery import launch_recovery_actionruns_for_job_runs from tron.utils import timeutils class TestRecovery(TestCase): @setup def fake_action_runs(self): mock_unknown_machine = Mock(autospec=True) mock_ok_machine = Mock(autospec=True) mock_unknown_machine.state = ActionRun.UNKNOWN mock_ok_machine.state = ActionRun.SUCCEEDED self.action_runs = [ SSHActionRun( job_run_id="test.unknown", name="test.unknown", node=Mock(), command_config=Mock(), machine=mock_unknown_machine, end_time=timeutils.current_time(), ), SSHActionRun( job_run_id="test.succeeded", name="test.succeeded", node=Mock(), command_config=Mock(), machine=mock_ok_machine, ), MesosActionRun( job_run_id="test.succeeded", name="test.succeeded", node=Mock(), command_config=Mock(), machine=mock_ok_machine, ), MesosActionRun( job_run_id="test.unknown-mesos", name="test.unknown-mesos", node=Mock(), command_config=Mock(), machine=mock_unknown_machine, ), MesosActionRun( job_run_id="test.unknown-mesos-done", name="test.unknown-mesos-done", node=Mock(), command_config=Mock(), machine=mock_unknown_machine, end_time=timeutils.current_time(), ), # TODO: Convert to all KubernetesActionRuns after deprecating mesos # A job will normally only ever have MesosActionRuns or KubernetsActionRuns KubernetesActionRun( job_run_id="test.k8s-done", name="test.k8s-done", node=Mock(), command_config=Mock(), machine=mock_unknown_machine, end_time=timeutils.current_time(), ), KubernetesActionRun( job_run_id="test.k8s-unknown", name="test.k8s-unknown", node=Mock(), command_config=Mock(), machine=mock_unknown_machine, ), ] def test_filter_action_runs_needing_recovery(self): assert filter_action_runs_needing_recovery(self.action_runs) == ( [self.action_runs[0]], [self.action_runs[3]], [self.action_runs[6]], ) @mock.patch("tron.core.recovery.filter_action_runs_needing_recovery", autospec=True) def test_launch_recovery_actionruns_for_job_runs(self, mock_filter): mock_actions = ( [ mock.Mock( action_runner=NoActionRunnerFactory(), spec=SSHActionRun, ), mock.Mock( action_runner=SubprocessActionRunnerFactory( status_path="/tmp/foo", exec_path=("/tmp/foo"), ), spec=SSHActionRun, ), ], [ mock.Mock( action_runner=NoActionRunnerFactory(), spec=MesosActionRun, ), ], [ mock.Mock( action_runner=NoActionRunnerFactory(), spec=KubernetesActionRun, ), ], ) mock_filter.return_value = mock_actions mock_action_runner = mock.Mock(autospec=True) mock_job_run = mock.Mock() launch_recovery_actionruns_for_job_runs( [mock_job_run], mock_action_runner, ) ssh_runs = mock_actions[0] for run in ssh_runs: assert run.recover.call_count == 1 mesos_run = mock_actions[1][0] assert mesos_run.recover.call_count == 1 kubernetes_run = mock_actions[2][0] assert kubernetes_run.recover.call_count == 1 @mock.patch("tron.core.recovery.filter_action_runs_needing_recovery", autospec=True) def test_launch_recovery_actionruns_empty_job_run(self, mock_filter): """_action_runs=None shouldn't prevent other job runs from being recovered""" empty_job_run = mock.Mock(_action_runs=None) other_job_run = mock.Mock(_action_runs=[mock.Mock()]) mock_action_runner = mock.Mock() mock_filter.return_value = ([], [], []) launch_recovery_actionruns_for_job_runs( [empty_job_run, other_job_run], mock_action_runner, ) mock_filter.assert_called_with(other_job_run._action_runs) ================================================ FILE: tests/data/logging.conf ================================================ [loggers] keys=root, twisted, tron [handlers] keys=fileHandler [formatters] keys=defaultFormatter [logger_root] level=WARN handlers=fileHandler [logger_twisted] level=WARN handlers=fileHandler qualname=twisted propagate=0 [logger_tron] level=WARN handlers=fileHandler qualname=tron propagate=0 [handler_fileHandler] class=logging.FileHandler level=WARN formatter=defaultFormatter args=('{0}',) [formatter_defaultFormatter] format=%(asctime)s %(name)s %(levelname)s %(message)s ================================================ FILE: tests/data/test_config.yaml ================================================ # This test config is intended to cover most common configuration cases ssh_options: agent: true state_persistence: store_type: shelve name: tron_state.shelve buffer_size: 1 nodes: - hostname: localhost - name: box1 hostname: localhost - name: box2 hostname: localhost - name: box3 hostname: localhost - name: box4 hostname: 127.0.0.1 node_pools: - name: pool0 nodes: [localhost, box2] - name: pool1 nodes: [box1] - nodes: [box1, box2] - name: pool2 nodes: [box3, box4] command_context: THE_JOB_DIR: "testconfig/jobs" ECHO: "echo" # Change this to repo root PYTHON: "cd /home/user/code/Tron && PYTHONPATH=. python" jobs: # IntervalScheduler no dependent Actions, single node - name: interval_job0 run_limit: 3 node: localhost schedule: "cron * * * * *" actions: - name: "task0" command: "%(ECHO)s %(actionname)s" - name: "task1" command: "sleep 10 && %(ECHO)s %(actionname)s" # IntervalScheduler dependent successful Actions, node pool - name: interval_job1 node: pool0 schedule: "cron * * * * *" actions: - name: task1 command: "%(ECHO)s %(actionname)s" requires: [task0] - name: task0 command: "sleep 3 && %(ECHO)s %(actionname)s %(last_success:shortdate)s" # IntervalScheduler dependent failure Actions - name: interval_job2 node: box1_box2 schedule: "cron * * * * *" actions: - name: task1 command: "%(ECHO)s %(actionname)s" requires: [task0] - name: task0 command: "%(ECHO)s %(actionname)s && sleep 7 && false" # Multiple dependent failure Actions - name: interval_job3 node: box1_box2 schedule: "cron * * * * *" actions: - name: task0 command: "%(ECHO)s %(actionname)s && sleep 7 && false" - name: task1 command: "%(ECHO)s %(actionname)s" requires: [task0] - name: task2 command: "%(ECHO)s %(actionname)s && sleep 10" - name: task3 command: "%(ECHO)s %(actionname)s" requires: [task2] # Multiple dependent failure Actions with cleanup - name: interval_job4 node: box2 schedule: "cron * * * * *" actions: - name: task0 command: "%(ECHO)s %(actionname)s && sleep 7 && false" - name: task1 command: "%(ECHO)s %(actionname)s" requires: [task0] - name: task2 command: "%(ECHO)s %(actionname)s && sleep 10" - name: task3 command: "%(ECHO)s %(actionname)s && sleep 3" requires: [task2] cleanup_action: command: "%(ECHO)s %(actionname)s %(cleanup_job_status)s" # No failures, with cleanup, different node for action - name: interval_job5 node: box1 schedule: "cron * * * * *" actions: - name: "task0" command: "%(ECHO)s %(actionname)s" node: box2 - name: "task1" command: "sleep 10 && %(ECHO)s %(actionname)s" node: pool0 cleanup_action: command: "%(ECHO)s %(actionname)s %(cleanup_job_status)s" # all_nodes Job - name: allnodes_job8 node: pool2 schedule: "cron * * * * *" all_nodes: true actions: - name: "task0" command: "%(ECHO)s %(actionname)s" - name: "task1" command: "sleep 10 && %(ECHO)s %(actionname)s" # Job failing bad action # DailyScheduler - name: daily_job9 node: box1 schedule: "daily 16:00:00" actions: - name: "task0" command: "%(ECHO)s %(actionname)s 1 && false" node: box2 requires: ["task1"] - name: "task1" command: "sleep 10 && %(ECHO)s %(actionname)s %(last_success:shortdate)s" node: pool0 cleanup_action: command: "%(ECHO)s %(actionname)s %(cleanup_job_status)s" # Overlapping, queueing - name: overlap_cancel node: pool2 schedule: "cron * * * * *" queueing: false actions: - name: "task0" command: "sleep 30s && %(ECHO)s %(actionname)s" ================================================ FILE: tests/eventbus_test.py ================================================ import os import tempfile from collections import defaultdict from unittest import mock from testifycompat import assert_equal from testifycompat import setup from testifycompat import teardown from testifycompat import TestCase from tron.eventbus import EventBus class MakeEventBusTestCase(TestCase): @setup def setup(self): self.logdir = tempfile.TemporaryDirectory() @teardown def teardown(self): EventBus.shutdown() self.logdir.cleanup() @mock.patch("tron.eventbus.time", autospec=True) def test_setup_eventbus_dir(self, time): os.rmdir(self.logdir.name) time.time = mock.Mock(return_value=1.0) eb = EventBus.create(self.logdir.name) assert os.path.exists(self.logdir.name) assert os.path.exists(os.path.join(self.logdir.name, "current")) time.time = mock.Mock(return_value=2.0) eb.event_log = {"foo": "bar"} eb.sync_save_log("test") new_eb = EventBus.create(self.logdir.name) new_eb.sync_load_log() assert new_eb.event_log == eb.event_log class EventBusTestCase(TestCase): @setup def setup(self): self.log_dir = tempfile.TemporaryDirectory(prefix="tron_eventbus_test") self.eventbus = EventBus.create(self.log_dir.name) self.eventbus.enabled = True @teardown def teardown(self): EventBus.shutdown() self.log_dir.cleanup() @mock.patch("tron.eventbus.reactor", autospec=True) def test_start(self, reactor): self.eventbus.sync_load_log = mock.Mock() reactor.callLater = mock.Mock() self.eventbus.start() assert self.eventbus.sync_load_log.call_count == 1 assert reactor.callLater.call_count == 1 def test_shutdown(self): assert self.eventbus.enabled self.eventbus.sync_save_log = mock.Mock() self.eventbus.shutdown() assert not self.eventbus.enabled assert self.eventbus.sync_save_log.call_count == 1 def test_publish(self): evt = {"id": "foo"} self.eventbus.publish(evt) assert self.eventbus.publish_queue.pop() == evt def test_subscribe(self): ps = ("foo", "bar", "cb") self.eventbus.subscribe(*ps) assert self.eventbus.subscribe_queue.pop() == ps def test_has_event(self): assert not self.eventbus.has_event("foo") self.eventbus.event_log["foo"] = "bar" assert self.eventbus.has_event("foo") @mock.patch("tron.eventbus.time", autospec=True) def test_sync_load_log(self, time): time.time = mock.Mock(return_value=1.0) self.eventbus.event_log = {"foo": "bar"} self.eventbus.sync_save_log("test") self.eventbus.event_log = {} self.eventbus.sync_load_log() assert self.eventbus.event_log == {"foo": "bar"} @mock.patch("tron.eventbus.time", autospec=True) def test_sync_save_log_time(self, time): time.time = mock.Mock(return_value=1.0) self.eventbus.sync_save_log("test") current_link = os.readlink(self.eventbus.log_current) assert_equal(current_link, os.path.join(self.log_dir.name, "1.pickle")) time.time = mock.Mock(return_value=2.0) self.eventbus.sync_save_log("test") new_link = os.readlink(self.eventbus.log_current) assert_equal(new_link, os.path.join(self.log_dir.name, "2.pickle")) # we clean up the previous link so as not to have a million pickles # on disk assert not os.path.exists(current_link) # so at this point, we should only have the new link assert os.path.exists(new_link) @mock.patch("tron.eventbus.time", autospec=True) @mock.patch("tron.eventbus.reactor", autospec=True) def test_sync_loop(self, reactor, time): time.time = mock.Mock(return_value=0) reactor.callLater = mock.Mock() self.eventbus.enabled = True self.eventbus.sync_shutdown = mock.Mock() self.eventbus.sync_loop() assert reactor.callLater.call_count == 1 assert self.eventbus.sync_shutdown.call_count == 0 @mock.patch("tron.eventbus.reactor", autospec=True) def test_sync_loop_shutdown(self, reactor): reactor.callLater = mock.Mock() self.eventbus.enabled = False self.eventbus.sync_save_log = mock.Mock() self.eventbus.sync_loop() assert reactor.callLater.call_count == 0 @mock.patch("tron.eventbus.time", autospec=True) def test_sync_process_save_log(self, time): time.time = mock.Mock(return_value=10) self.eventbus.log_updates = 1 self.eventbus.log_last_save = 0 self.eventbus.log_save_interval = 20 self.eventbus.sync_save_log = mock.Mock() self.eventbus.sync_process() assert self.eventbus.sync_save_log.call_count == 0 time.time = mock.Mock(return_value=21) self.eventbus.sync_process() assert self.eventbus.sync_save_log.call_count == 1 time.time = mock.Mock(return_value=0) self.eventbus.log_updates = 0 self.eventbus.log_save_updates = 20 self.eventbus.sync_save_log = mock.Mock() self.eventbus.sync_process() assert self.eventbus.sync_save_log.call_count == 0 self.eventbus.log_updates = 21 self.eventbus.sync_process() assert self.eventbus.sync_save_log.call_count == 1 assert self.eventbus.log_updates == 0 @mock.patch("tron.eventbus.time", autospec=True) def test_sync_process_flush_queues(self, time): time.time = mock.Mock(return_value=10) self.eventbus.sync_subscribe = mock.Mock() self.eventbus.sync_publish = mock.Mock() for _ in range(5): self.eventbus.publish_queue.append(mock.Mock()) self.eventbus.subscribe_queue.append(mock.Mock()) self.eventbus.sync_process() assert_equal(self.eventbus.sync_subscribe.call_count, 5) assert_equal(self.eventbus.sync_publish.call_count, 5) @mock.patch("tron.eventbus.reactor", autospec=True) def test_sync_publish(self, reactor): reactor.callLater = mock.Mock() evt = {"id": "foo", "bar": "baz"} self.eventbus.event_log = {} self.eventbus.log_save_updates = 0 self.eventbus.sync_publish(evt) assert self.eventbus.log_updates == 1 assert reactor.callLater.call_count == 1 @mock.patch("tron.eventbus.reactor", autospec=True) def test_sync_publish_replace(self, reactor): evt1 = {"id": "foo", "bar": "baz"} evt2 = {"id": "foo", "bar": "quux"} self.eventbus.event_log = {} self.eventbus.log_save_updates = 0 self.eventbus.sync_publish(evt1) self.eventbus.sync_publish(evt2) assert self.eventbus.log_updates == 2 assert reactor.callLater.call_count == 2 @mock.patch("tron.eventbus.reactor", autospec=True) def test_sync_publish_duplicate(self, reactor): evt = {"id": "foo", "bar": "baz"} self.eventbus.event_log = {"foo": {"bar": "baz"}} self.eventbus.log_save_updates = 0 self.eventbus.sync_publish(evt) assert self.eventbus.log_updates == 0 assert reactor.callLater.call_count == 0 def test_sync_subscribe(self): self.eventbus.event_subscribers = defaultdict(list) self.eventbus.sync_subscribe(("pre", "sub", "cb")) assert self.eventbus.event_subscribers == {"pre": [("sub", "cb")]} self.eventbus.sync_subscribe(("pre", "sub2", "cb2")) assert self.eventbus.event_subscribers == { "pre": [("sub", "cb"), ("sub2", "cb2")], } def test_sync_unsubscribe(self): self.eventbus.event_subscribers = defaultdict(list) self.eventbus.sync_subscribe(("pre", "sub", "cb")) self.eventbus.sync_subscribe(("pre", "sub2", "cb2")) assert self.eventbus.event_subscribers == { "pre": [("sub", "cb"), ("sub2", "cb2")], } self.eventbus.sync_unsubscribe(("pre", "sub")) assert self.eventbus.event_subscribers == {"pre": [("sub2", "cb2")]} self.eventbus.sync_unsubscribe(("pre", "sub2")) assert self.eventbus.event_subscribers == {} @mock.patch("tron.eventbus.reactor", autospec=True) def test_sync_notify(self, reactor): reactor.callLater = mock.Mock() self.eventbus.event_log = {"p": {}, "pre": {}, "prefix": {}} self.eventbus.event_subscribers = { "pre": [("sub", "m1")], "prefix": [("sub", "m2"), ("sub2", "m3")], } self.eventbus.sync_notify("p") assert reactor.callLater.call_count == 0 self.eventbus.sync_notify("pre") assert reactor.callLater.call_count == 1 self.eventbus.sync_notify("prefix") assert reactor.callLater.call_count == 4 ================================================ FILE: tests/kubernetes_test.py ================================================ from typing import Any from unittest import mock import pytest from task_processing.interfaces.event import Event from task_processing.plugins.kubernetes.task_config import KubernetesTaskConfig from tron.config.schema import ConfigFieldSelectorSource from tron.config.schema import ConfigKubernetes from tron.config.schema import ConfigProjectedSAVolume from tron.config.schema import ConfigSecretSource from tron.config.schema import ConfigSecretVolume from tron.config.schema import ConfigSecretVolumeItem from tron.config.schema import ConfigVolume from tron.kubernetes import DEFAULT_DISK_LIMIT from tron.kubernetes import KubernetesCluster from tron.kubernetes import KubernetesClusterRepository from tron.kubernetes import KubernetesTask from tron.utils import exitcode @pytest.fixture def mock_kubernetes_task(): with mock.patch( "tron.kubernetes.logging.getLogger", return_value=mock.Mock(handlers=[mock.Mock()]), autospec=None, ): yield KubernetesTask( action_run_id="mock_service.mock_job.1.mock_action", task_config=KubernetesTaskConfig( name="mock--service-mock-job-mock--action", uuid="123456", image="some_image", command="echo test" ), ) @pytest.fixture def mock_kubernetes_cluster(): with mock.patch("tron.kubernetes.PyDeferredQueue", autospec=True,), mock.patch( "tron.kubernetes.TaskProcessor", autospec=True, ), mock.patch( "tron.kubernetes.Subscription", autospec=True, ) as mock_runner: mock_runner.return_value.configure_mock( stopping=False, TASK_CONFIG_INTERFACE=mock.Mock(spec=KubernetesTaskConfig) ) yield KubernetesCluster("kube-cluster-a:1234") @pytest.fixture def mock_disabled_kubernetes_cluster(): with mock.patch("tron.kubernetes.PyDeferredQueue", autospec=True,), mock.patch( "tron.kubernetes.TaskProcessor", autospec=True, ), mock.patch( "tron.kubernetes.Subscription", autospec=True, ): yield KubernetesCluster("kube-cluster-a:1234", enabled=False) def mock_event_factory( task_id: str, platform_type: str, message: str = None, raw: dict[str, Any] = None, success: bool = False, terminal: bool = False, ) -> Event: return Event( kind="task", task_id=task_id, platform_type=platform_type, raw=raw or {}, terminal=terminal, success=success, message=message, ) def test_get_event_logger_add_unique_handlers(mock_kubernetes_task): """ Ensures that only a single handler (for stderr) is added to the Kubernetes Taskevent logger, to prevent duplicate log output. """ # Call 2 times to make sure 2nd call doesn't add another handler logger = mock_kubernetes_task.get_event_logger() logger = mock_kubernetes_task.get_event_logger() assert len(logger.handlers) == 1 def test_handle_event_log_event_info_exception(mock_kubernetes_task): with mock.patch.object( mock_kubernetes_task, "log_event_info", autospec=True, side_effect=Exception ) as mock_log_event_info: mock_kubernetes_task.handle_event( mock_event_factory(task_id=mock_kubernetes_task.get_kubernetes_id(), platform_type="running") ) # TODO: should also assert that the task is in the expected state once that's hooked up assert mock_log_event_info.called def test_handle_event_exit_early_on_misrouted_event(mock_kubernetes_task): with mock.patch.object( mock_kubernetes_task, "log_event_info", autospec=True, ) as mock_log_event_info: mock_kubernetes_task.handle_event( mock_event_factory(task_id="not-the-pods-youre-looking-for", platform_type="finished") ) # TODO: should also assert that the task is in the expected state once that's hooked up # we log before actually doing anything with an event, so this not being called means # we exited early assert not mock_log_event_info.called def test_handle_event_running(mock_kubernetes_task): mock_kubernetes_task.handle_event( mock_event_factory(task_id=mock_kubernetes_task.get_kubernetes_id(), platform_type="running") ) assert mock_kubernetes_task.state == mock_kubernetes_task.RUNNING def test_handle_event_exit_on_finished(mock_kubernetes_task): mock_kubernetes_task.started() raw_event_data = { "status": { "containerStatuses": [ { "containerID": "docker://asdf", "image": "someimage", "imageID": "docker-pullable://someimage:sometag", "lastState": {"running": None, "terminated": None, "waiting": None}, "name": "main", "ready": False, "restartCount": 0, "started": False, "state": { "running": None, "terminated": { "containerID": "docker://asdf", "exitCode": 0, "finishedAt": "2022-11-19 00:11:02+00:00", "message": None, "reason": "Completed", "signal": None, "startedAt": None, }, "waiting": None, }, }, ], } } mock_kubernetes_task.handle_event( mock_event_factory( task_id=mock_kubernetes_task.get_kubernetes_id(), raw=raw_event_data, platform_type="finished", terminal=True, success=True, ) ) assert mock_kubernetes_task.state == mock_kubernetes_task.COMPLETE assert mock_kubernetes_task.is_complete def test_handle_event_exit_on_failed(mock_kubernetes_task): mock_kubernetes_task.started() mock_kubernetes_task.handle_event( mock_event_factory( task_id=mock_kubernetes_task.get_kubernetes_id(), platform_type="failed", terminal=True, success=False ) ) assert mock_kubernetes_task.is_failed assert mock_kubernetes_task.is_done def test_handle_event_spot_interruption_exit(mock_kubernetes_task): mock_kubernetes_task.started() raw_event_data = { "status": { "containerStatuses": [ { "containerID": None, "image": "someimage", "imageID": None, "lastState": { "running": None, "terminated": { "containerID": None, "exitCode": 137, "finishedAt": None, "message": "The container could not be located when the pod was deleted. The container used to be Running", "reason": "ContainerStatusUnknown", "signal": None, "startedAt": None, }, "waiting": None, }, "name": "main", "ready": False, "restartCount": 0, "started": False, "state": { "running": None, "terminated": None, "waiting": {"message": None, "reason": "ContainerCreating"}, }, }, ], } } mock_kubernetes_task.handle_event( mock_event_factory( task_id=mock_kubernetes_task.get_kubernetes_id(), raw=raw_event_data, platform_type="killed", terminal=True, success=False, ) ) assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_SPOT_INTERRUPTION assert mock_kubernetes_task.is_failed assert mock_kubernetes_task.is_done # Test again, but with no lastState raw_event_data["status"]["containerStatuses"][0]["state"]["terminated"] = raw_event_data["status"][ "containerStatuses" ][0]["lastState"]["terminated"] raw_event_data["status"]["containerStatuses"][0]["lastState"] = {} mock_kubernetes_task.handle_event( mock_event_factory( task_id=mock_kubernetes_task.get_kubernetes_id(), raw=raw_event_data, platform_type="killed", terminal=True, success=False, ) ) assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_SPOT_INTERRUPTION assert mock_kubernetes_task.is_failed assert mock_kubernetes_task.is_done def test_handle_event_node_scaledown_exit(mock_kubernetes_task): mock_kubernetes_task.started() raw_event_data = { "status": { "containerStatuses": [ { "containerID": "docker://asdf", "image": "someimage", "imageID": "docker-pullable://someimage:sometag", "lastState": {"running": None, "terminated": None, "waiting": None}, "name": "main", "ready": False, "restartCount": 0, "started": False, "state": { "running": None, "terminated": { "containerID": "docker://asdf", "exitCode": 143, "finishedAt": "2022-11-19 00:11:02+00:00", "message": None, "reason": "Error", "signal": None, "startedAt": None, }, "waiting": None, }, }, ], } } mock_kubernetes_task.handle_event( mock_event_factory( task_id=mock_kubernetes_task.get_kubernetes_id(), raw=raw_event_data, platform_type="failed", terminal=True, success=False, ) ) assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_NODE_SCALEDOWN assert mock_kubernetes_task.is_failed assert mock_kubernetes_task.is_done def test_handle_event_exit_not_terminated(mock_kubernetes_task): mock_kubernetes_task.started() raw_event_data = { "status": { "containerStatuses": [ { "containerID": "docker://asdf", "image": "someimage", "imageID": "docker-pullable://someimage:sometag", "lastState": {}, "name": "main", "ready": False, "restartCount": 0, "started": False, "state": { "running": None, "terminated": None, "waiting": {"reason": "ContainerCreating"}, }, }, ], } } mock_kubernetes_task.handle_event( mock_event_factory( task_id=mock_kubernetes_task.get_kubernetes_id(), raw=raw_event_data, platform_type="killed", terminal=True, success=False, ) ) assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_NODE_SCALEDOWN assert mock_kubernetes_task.is_failed assert mock_kubernetes_task.is_done def test_handle_event_abnormal_exit(mock_kubernetes_task): mock_kubernetes_task.started() raw_event_data = { "status": { "containerStatuses": [ { "containerID": "docker://asdf", "image": "someimage", "imageID": "docker-pullable://someimage:sometag", "lastState": {"running": None, "terminated": None, "waiting": None}, "name": "main", "ready": False, "restartCount": 0, "started": False, "state": { "running": None, "terminated": { "containerID": "docker://asdf", "exitCode": 0, "finishedAt": None, "message": None, "reason": None, "signal": None, "startedAt": None, }, "waiting": None, }, }, ], } } mock_kubernetes_task.handle_event( mock_event_factory( task_id=mock_kubernetes_task.get_kubernetes_id(), raw=raw_event_data, platform_type="finished", terminal=True, success=False, ) ) assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_ABNORMAL assert mock_kubernetes_task.is_failed assert mock_kubernetes_task.is_done def test_handle_event_missing_state(mock_kubernetes_task): mock_kubernetes_task.started() raw_event_data = { "status": { "containerStatuses": [ { "containerID": "docker://asdf", "image": "someimage", "imageID": "docker-pullable://someimage:sometag", "lastState": {}, "name": "main", "ready": False, "restartCount": 0, "started": False, "state": None, }, ], } } mock_kubernetes_task.handle_event( mock_event_factory( task_id=mock_kubernetes_task.get_kubernetes_id(), raw=raw_event_data, platform_type="killed", terminal=True, success=False, ) ) assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_ABNORMAL assert mock_kubernetes_task.is_failed assert mock_kubernetes_task.is_done def test_handle_event_code_from_state(mock_kubernetes_task): mock_kubernetes_task.started() raw_event_data = { "status": { "containerStatuses": [ { "containerID": "docker://asdf", "image": "someimage", "imageID": "docker-pullable://someimage:sometag", "lastState": {}, "name": "main", "ready": False, "restartCount": 0, "started": False, "state": { "running": None, "terminated": { "containerID": "docker://asdf", "exitCode": 1337, "finishedAt": None, "message": None, "reason": None, "signal": None, "startedAt": None, }, "waiting": None, }, }, ], } } mock_kubernetes_task.handle_event( mock_event_factory( task_id=mock_kubernetes_task.get_kubernetes_id(), raw=raw_event_data, platform_type="failed", terminal=True, success=False, ) ) assert mock_kubernetes_task.exit_status == 1337 assert mock_kubernetes_task.is_failed assert mock_kubernetes_task.is_done def test_handle_event_lost(mock_kubernetes_task): mock_kubernetes_task.started() mock_kubernetes_task.handle_event( mock_event_factory( task_id=mock_kubernetes_task.get_kubernetes_id(), platform_type="lost", ) ) assert mock_kubernetes_task.exit_status == exitcode.EXIT_KUBERNETES_TASK_LOST def test_create_task_disabled(): cluster = KubernetesCluster("kube-cluster-a:1234", enabled=False) mock_serializer = mock.MagicMock() task = cluster.create_task( action_run_id="action_a", serializer=mock_serializer, command="ls", cpus=1, mem=1024, disk=None, docker_image="docker-paasta.yelpcorp.com:443/bionic_yelp", env={}, secret_env={}, secret_volumes=[], projected_sa_volumes=[], field_selector_env={}, volumes=[], cap_add=[], cap_drop=[], node_selectors={"yelp.com/pool": "default"}, node_affinities=[], topology_spread_constraints=[], pod_labels={}, pod_annotations={}, service_account_name=None, ports=[], ) assert task is None def test_create_task(mock_kubernetes_cluster): mock_serializer = mock.MagicMock() task = mock_kubernetes_cluster.create_task( action_run_id="action_a", serializer=mock_serializer, command="ls", cpus=1, mem=1024, disk=None, docker_image="docker-paasta.yelpcorp.com:443/bionic_yelp", env={}, secret_env={}, secret_volumes=[], projected_sa_volumes=[], field_selector_env={}, volumes=[], cap_add=[], cap_drop=[], node_selectors={"yelp.com/pool": "default"}, node_affinities=[], topology_spread_constraints=[], pod_labels={}, pod_annotations={}, service_account_name=None, ports=[], ) assert task is not None def test_create_task_with_task_id(mock_kubernetes_cluster): mock_serializer = mock.MagicMock() task = mock_kubernetes_cluster.create_task( action_run_id="action_a", serializer=mock_serializer, task_id="yay.1234", command="ls", cpus=1, mem=1024, disk=None, docker_image="docker-paasta.yelpcorp.com:443/bionic_yelp", env={}, secret_env={}, secret_volumes=[], projected_sa_volumes=[], field_selector_env={}, volumes=[], cap_add=[], cap_drop=[], node_selectors={"yelp.com/pool": "default"}, node_affinities=[], topology_spread_constraints=[], pod_labels={}, pod_annotations={}, service_account_name=None, ports=[], ) mock_kubernetes_cluster.runner.TASK_CONFIG_INTERFACE().set_pod_name.assert_called_once_with("yay.1234") assert task is not None def test_create_task_with_invalid_task_id(mock_kubernetes_cluster): mock_serializer = mock.MagicMock() with mock.patch.object(mock_kubernetes_cluster, "runner") as mock_runner: mock_runner.TASK_CONFIG_INTERFACE.return_value.set_pod_name = mock.MagicMock(side_effect=ValueError) task = mock_kubernetes_cluster.create_task( action_run_id="action_a", serializer=mock_serializer, task_id="boo", command="ls", cpus=1, mem=1024, disk=None, docker_image="docker-paasta.yelpcorp.com:443/bionic_yelp", env={}, secret_env={}, secret_volumes=[], projected_sa_volumes=[], field_selector_env={}, volumes=[], cap_add=[], cap_drop=[], node_selectors={"yelp.com/pool": "default"}, node_affinities=[], topology_spread_constraints=[], pod_labels={}, pod_annotations={}, service_account_name=None, ports=[], ) assert task is None def test_create_task_with_config(mock_kubernetes_cluster): # Validate we pass all expected args to taskproc default_volumes = [ConfigVolume(container_path="/nail/tmp", host_path="/nail/tmp", mode="RO")] mock_kubernetes_cluster.default_volumes = default_volumes mock_serializer = mock.MagicMock() config_volumes = [ConfigVolume(container_path="/tmp", host_path="/host", mode="RO")] config_secret_volumes = [ ConfigSecretVolume( secret_volume_name="secretvolumename", secret_name="secret", container_path="/b", default_mode="0644", items=[ConfigSecretVolumeItem(key="key", path="path", mode="0755")], ), ] config_secrets = {"TEST_SECRET": ConfigSecretSource(secret_name="tron-secret-test-secret--A", key="secret_A")} config_field_selector = {"POD_IP": ConfigFieldSelectorSource(field_path="status.podIP")} config_sa_volumes = [ConfigProjectedSAVolume(audience="for.bar.com", container_path="/var/run/secrets/whatever")] expected_args = { "name": mock.ANY, "command": "ls", "image": "docker-paasta.yelpcorp.com:443/bionic_yelp", "cpus": 1, "memory": 1024, "disk": DEFAULT_DISK_LIMIT, "environment": {"TEST_ENV": "foo"}, "secret_environment": {k: v._asdict() for k, v in config_secrets.items()}, "secret_volumes": [v._asdict() for v in config_secret_volumes], "projected_sa_volumes": [v._asdict() for v in config_sa_volumes], "field_selector_environment": {k: v._asdict() for k, v in config_field_selector.items()}, "volumes": [v._asdict() for v in default_volumes + config_volumes], "cap_add": ["KILL"], "cap_drop": ["KILL", "CHOWN"], "node_selectors": {"yelp.com/pool": "default"}, "node_affinities": [], "topology_spread_constraints": [], "labels": {}, "annotations": {}, "service_account_name": None, "ports": [], } task = mock_kubernetes_cluster.create_task( action_run_id="action_a", serializer=mock_serializer, task_id="yay.1234", command=expected_args["command"], cpus=expected_args["cpus"], mem=expected_args["memory"], disk=None, docker_image=expected_args["image"], env=expected_args["environment"], secret_env=config_secrets, secret_volumes=config_secret_volumes, projected_sa_volumes=config_sa_volumes, field_selector_env=config_field_selector, volumes=config_volumes, cap_add=["KILL"], cap_drop=["KILL", "CHOWN"], node_selectors={"yelp.com/pool": "default"}, node_affinities=[], topology_spread_constraints=[], pod_labels={}, pod_annotations={}, service_account_name=None, ports=expected_args["ports"], ) assert task is not None mock_kubernetes_cluster.runner.TASK_CONFIG_INTERFACE.assert_called_once_with(**expected_args) def test_process_event_task(mock_kubernetes_cluster): event = mock_event_factory(task_id="abc.123", platform_type="mock_type") mock_kubernetes_task = mock.MagicMock(spec_set=KubernetesTask) mock_kubernetes_task.get_kubernetes_id.return_value = "abc.123" mock_kubernetes_cluster.tasks["abc.123"] = mock_kubernetes_task mock_kubernetes_cluster.process_event(event) mock_kubernetes_task.handle_event.assert_called_once_with(event) def test_process_event_task_invalid_id(mock_kubernetes_cluster): event = mock_event_factory(task_id="hwat.dis", platform_type="mock_type") mock_kubernetes_task = mock.MagicMock(spec_set=KubernetesTask) mock_kubernetes_task.get_kubernetes_id.return_value = "abc.123" mock_kubernetes_cluster.tasks["abc.123"] = mock_kubernetes_task mock_kubernetes_cluster.process_event(event) assert mock_kubernetes_task.handle_event.call_count == 0 def test_stop_default(mock_kubernetes_cluster): # When stopping, tasks should not exit. They will be recovered mock_task = mock.MagicMock() mock_kubernetes_cluster.tasks = {"task_id": mock_task} mock_kubernetes_cluster.stop() assert mock_kubernetes_cluster.deferred is None assert mock_task.exited.call_count == 0 assert len(mock_kubernetes_cluster.tasks) == 1 def test_stop_disabled(): # Shouldn't raise an error mock_kubernetes_cluster = KubernetesCluster("kube-cluster-a:1234", enabled=False) mock_kubernetes_cluster.stop() def test_set_enabled_enable_already_on(mock_kubernetes_cluster): mock_kubernetes_cluster.set_enabled(is_enabled=True) assert mock_kubernetes_cluster.enabled is True # only called once as part of creating the cluster object mock_kubernetes_cluster.processor.executor_from_config.assert_called_once() assert mock_kubernetes_cluster.runner is not None assert mock_kubernetes_cluster.deferred is not None mock_kubernetes_cluster.deferred.addCallback.assert_has_calls( [ mock.call(mock_kubernetes_cluster.process_event), mock.call(mock_kubernetes_cluster.handle_next_event), ] ) def test_set_enabled_enable(mock_disabled_kubernetes_cluster): mock_disabled_kubernetes_cluster.set_enabled(is_enabled=True) assert mock_disabled_kubernetes_cluster.enabled is True # only called once as part of enabling mock_disabled_kubernetes_cluster.processor.executor_from_config.assert_called_once() assert mock_disabled_kubernetes_cluster.runner is not None assert mock_disabled_kubernetes_cluster.deferred is not None mock_disabled_kubernetes_cluster.deferred.addCallback.assert_has_calls( [ mock.call(mock_disabled_kubernetes_cluster.process_event), mock.call(mock_disabled_kubernetes_cluster.handle_next_event), ] ) def test_set_enabled_disable(mock_kubernetes_cluster): mock_task = mock.Mock(spec=KubernetesTask) mock_kubernetes_cluster.tasks == {"a.b": mock_task} mock_kubernetes_cluster.set_enabled(is_enabled=False) assert mock_kubernetes_cluster.enabled is False mock_kubernetes_cluster.runner.stop.assert_called_once() assert mock_kubernetes_cluster.deferred is None assert mock_kubernetes_cluster.tasks == {} def test_configure_default_volumes(): # default_volume validation is done at config time, we just need to validate we are setting it with mock.patch("tron.kubernetes.PyDeferredQueue", autospec=True,), mock.patch( "tron.kubernetes.TaskProcessor", autospec=True, ), mock.patch( "tron.kubernetes.Subscription", autospec=True, ): mock_kubernetes_cluster = KubernetesCluster("kube-cluster-a:1234", default_volumes=[]) assert mock_kubernetes_cluster.default_volumes == [] expected_volumes = [ ConfigVolume( container_path="/tmp", host_path="/host/tmp", mode="RO", ), ] mock_kubernetes_cluster.configure_tasks(default_volumes=expected_volumes) assert mock_kubernetes_cluster.default_volumes == expected_volumes def test_submit_disabled(mock_disabled_kubernetes_cluster, mock_kubernetes_task): with mock.patch.object(mock_kubernetes_task, "exited", autospec=True) as mock_exited: mock_disabled_kubernetes_cluster.submit(mock_kubernetes_task) assert mock_kubernetes_task.get_kubernetes_id() not in mock_disabled_kubernetes_cluster.tasks mock_exited.assert_called_once_with(1) def test_submit(mock_kubernetes_cluster, mock_kubernetes_task): mock_kubernetes_cluster.submit(mock_kubernetes_task) assert mock_kubernetes_task.get_kubernetes_id() in mock_kubernetes_cluster.tasks assert mock_kubernetes_cluster.tasks[mock_kubernetes_task.get_kubernetes_id()] == mock_kubernetes_task mock_kubernetes_cluster.runner.run.assert_called_once_with(mock_kubernetes_task.get_config()) def test_recover(mock_kubernetes_cluster, mock_kubernetes_task): with mock.patch.object(mock_kubernetes_task, "started", autospec=True) as mock_started: mock_kubernetes_cluster.recover(mock_kubernetes_task) assert mock_kubernetes_task.get_kubernetes_id() in mock_kubernetes_cluster.tasks mock_kubernetes_cluster.runner.reconcile.assert_called_once_with(mock_kubernetes_task.get_config()) assert mock_started.call_count == 1 def test_kuberntes_cluster_repository(): # Check we are passing k8s_options from mcp/KubernetesClusterRepository.configure to KubernetesCluster calls mock_k8s_options = { "enabled": True, "kubeconfig_path": "/tmp/kubeconfig.conf", "watcher_kubeconfig_paths": ["/tmp/kubeconfig_old.conf"], "non_retryable_exit_codes": [13], "default_volumes": [ ConfigVolume( container_path="/tmp", host_path="/host/tmp", mode="RO", ) ], } mock_k8s_options_obj = ConfigKubernetes(**mock_k8s_options) with mock.patch("tron.kubernetes.KubernetesCluster", autospec=True) as mock_cluster: KubernetesClusterRepository.configure(mock_k8s_options_obj) KubernetesClusterRepository.get_cluster() mock_cluster.assert_called_once_with(**mock_k8s_options) ================================================ FILE: tests/mcp_reconfigure_test.py ================================================ """Tests for reconfiguring mcp.""" import os import tempfile import time import pytest from testifycompat import assert_equal from testifycompat import run from testifycompat import setup from testifycompat import suite from testifycompat import teardown from testifycompat import TestCase from tests.assertions import assert_length from tron import mcp from tron.config import config_parse from tron.config import schema from tron.serialize import filehandler class TestMCPReconfigure(TestCase): os.environ["SSH_AUTH_SOCK"] = "test-socket" pre_config = dict( ssh_options=dict( agent=True, identities=["tests/test_id_rsa"], ), nodes=[ dict(name="node0", hostname="batch0"), dict(name="node1", hostname="batch1"), ], node_pools=[dict(name="nodePool", nodes=["node0", "node1"])], command_context={ "thischanges": "froma", }, jobs=[ dict( name="test_unchanged", node="node0", schedule="daily", actions=[ dict( name="action_unchanged", command="command_unchanged", ), ], ), dict( name="test_remove", node="node1", schedule={"type": "cron", "value": "* * * * *"}, actions=[ dict( name="action_remove", command="command_remove", ), ], cleanup_action=dict(name="cleanup", command="doit"), ), dict( name="test_change", node="nodePool", schedule={"type": "cron", "value": "* * * * *"}, actions=[ dict( name="action_change", command="command_change", ), dict( name="action_remove2", command="command_remove2", requires=["action_change"], ), ], ), dict( name="test_daily_change", node="node0", schedule="daily", actions=[ dict( name="action_daily_change", command="command", ), ], ), dict( name="test_action_added", node="node0", schedule={"type": "cron", "value": "* * * * *"}, actions=[ dict(name="action_first", command="command_do_it"), ], ), ], ) post_config = dict( ssh_options=dict( agent=True, identities=["tests/test_id_rsa"], ), nodes=[ dict(name="node0", hostname="batch0"), dict(name="node1", hostname="batch1"), ], node_pools=[dict(name="nodePool", nodes=["node0", "node1"])], command_context={ "a_variable": "is_constant", "thischanges": "tob", }, jobs=[ dict( name="test_unchanged", node="node0", schedule="daily", actions=[ dict( name="action_unchanged", command="command_unchanged", ), ], ), dict( name="test_change", node="nodePool", schedule="daily", actions=[ dict( name="action_change", command="command_changed", ), ], ), dict( name="test_daily_change", node="node0", schedule="daily", actions=[ dict( name="action_daily_change", command="command_changed", ), ], ), dict( name="test_new", node="nodePool", schedule={"type": "cron", "value": "* * * * *"}, actions=[ dict( name="action_new", command="command_new", ), ], ), dict( name="test_action_added", node="node0", schedule={"type": "cron", "value": "* * * * *"}, actions=[ dict(name="action_first", command="command_do_it"), dict(name="action_second", command="command_ok"), ], ), ], ) def _get_config(self, idx, output_dir): config = dict(self.post_config if idx else self.pre_config) config["output_stream_dir"] = output_dir return config def _get_runs_to_schedule(self, sched): last_run = sched.job.runs.get_newest(include_manual=False) last_run_time = last_run.run_time if last_run else None return sched.get_runs_to_schedule(last_run_time) @setup def setup_mcp(self): self.test_dir = tempfile.mkdtemp() self.mcp = mcp.MasterControlProgram(self.test_dir, "config", time.time()) config = {schema.MASTER_NAMESPACE: self._get_config(0, self.test_dir)} container = config_parse.ConfigContainer.create(config) self.mcp.apply_config(container) @teardown def teardown_mcp(self): filehandler.OutputPath(self.test_dir).delete() filehandler.FileHandleManager.reset() def reconfigure(self): config = {schema.MASTER_NAMESPACE: self._get_config(1, self.test_dir)} container = config_parse.ConfigContainer.create(config) self.mcp.apply_config(container, reconfigure=True) @suite("integration") def test_job_list(self): count = len(self.pre_config["jobs"]) assert_equal(len(self.mcp.jobs.get_names()), count) self.reconfigure() assert_equal(len(self.mcp.jobs.get_names()), count) @pytest.mark.skip( reason="This test doesn't currently as run1 is not scheduled.", ) @suite("integration") def test_job_unchanged(self): assert "MASTER.test_unchanged" in self.mcp.jobs job_sched = self.mcp.jobs.get_by_name("MASTER.test_unchanged") orig_job = job_sched.job run0 = next(self._get_runs_to_schedule(job_sched)) run0.start() run1 = next(self._get_runs_to_schedule(job_sched)) assert_equal(job_sched.job.name, "MASTER.test_unchanged") action_map = job_sched.job.action_graph.action_map assert_equal(len(action_map), 1) assert_equal(action_map["action_unchanged"].name, "action_unchanged") assert_equal(str(job_sched.job.scheduler), "daily 00:00:00 ") self.reconfigure() assert job_sched is self.mcp.jobs.get_by_name("MASTER.test_unchanged") assert job_sched.job is orig_job assert_equal(len(job_sched.job.runs.runs), 2) assert_equal(job_sched.job.runs.runs[1], run0) assert_equal(job_sched.job.runs.runs[0], run1) assert run1.is_scheduled assert_equal(job_sched.job.context["a_variable"], "is_constant") assert_equal(job_sched.job.context["thischanges"], "tob") @suite("integration") def test_job_unchanged_disabled(self): job_sched = self.mcp.jobs.get_by_name("MASTER.test_unchanged") orig_job = job_sched.job next(self._get_runs_to_schedule(job_sched)) job_sched.disable() self.reconfigure() assert job_sched is self.mcp.jobs.get_by_name("MASTER.test_unchanged") assert job_sched.job is orig_job assert not job_sched.job.enabled @suite("integration") def test_job_removed(self): assert "MASTER.test_remove" in self.mcp.jobs job_sched = self.mcp.jobs.get_by_name("MASTER.test_remove") run0 = next(self._get_runs_to_schedule(job_sched)) run0.start() run1 = next(self._get_runs_to_schedule(job_sched)) assert_equal(job_sched.job.name, "MASTER.test_remove") action_map = job_sched.job.action_graph.action_map assert_equal(len(action_map), 2) assert_equal(action_map["action_remove"].name, "action_remove") self.reconfigure() assert "test_remove" not in self.mcp.jobs assert not job_sched.job.enabled assert not run1.is_scheduled @suite("integration") def test_job_changed(self): assert "MASTER.test_change" in self.mcp.jobs job_sched = self.mcp.jobs.get_by_name("MASTER.test_change") run0 = next(self._get_runs_to_schedule(job_sched)) run0.start() next(self._get_runs_to_schedule(job_sched)) assert_equal(len(job_sched.job.runs.runs), 2) assert_equal(job_sched.job.name, "MASTER.test_change") action_map = job_sched.job.action_graph.action_map assert_equal(len(action_map), 2) self.reconfigure() new_job_sched = self.mcp.jobs.get_by_name("MASTER.test_change") assert new_job_sched is job_sched assert new_job_sched.job is job_sched.job assert_equal(new_job_sched.job.name, "MASTER.test_change") action_map = job_sched.job.action_graph.action_map assert_equal(len(action_map), 1) assert_equal(len(new_job_sched.job.runs.runs), 2) assert new_job_sched.job.runs.runs[1].is_starting assert new_job_sched.job.runs.runs[0].is_scheduled assert_equal(job_sched.job.context["a_variable"], "is_constant") assert new_job_sched.job.context.base.job is new_job_sched.job @suite("integration") def test_job_changed_disabled(self): job_sched = self.mcp.jobs.get_by_name("MASTER.test_change") job_sched.disable() assert not job_sched.job.enabled self.reconfigure() new_job_sched = self.mcp.jobs.get_by_name("MASTER.test_change") assert not new_job_sched.job.enabled @suite("integration") def test_job_new(self): assert "test_new" not in self.mcp.jobs self.reconfigure() assert "MASTER.test_new" in self.mcp.jobs job_sched = self.mcp.jobs.get_by_name("MASTER.test_new") assert_equal(job_sched.job.name, "MASTER.test_new") action_map = job_sched.job.action_graph.action_map assert_equal(len(action_map), 1) assert_equal(action_map["action_new"].name, "action_new") assert_equal(action_map["action_new"].command, "command_new") assert_equal(len(job_sched.job.runs.runs), 1) assert job_sched.job.runs.runs[0].is_scheduled @suite("integration") def test_daily_reschedule(self): job_sched = self.mcp.jobs.get_by_name("MASTER.test_daily_change") next(self._get_runs_to_schedule(job_sched)) assert_equal(len(job_sched.job.runs.runs), 1) run = job_sched.job.runs.runs[0] assert run.is_scheduled action_runs = run.action_runs self.reconfigure() assert action_runs.is_cancelled assert_equal(len(job_sched.job.runs.runs), 1) new_run = job_sched.job.runs.runs[0] assert new_run is not run assert new_run.is_scheduled assert_equal(run.run_time, new_run.run_time) @suite("integration") def test_action_added(self): self.reconfigure() job_sched = self.mcp.jobs.get_by_name("MASTER.test_action_added") assert_length(job_sched.job.action_graph.action_map, 2) if __name__ == "__main__": run() ================================================ FILE: tests/mcp_test.py ================================================ import shutil import tempfile import time from unittest import mock import pytest from testifycompat import assert_equal from testifycompat import run from testifycompat import setup from testifycompat import teardown from testifycompat import TestCase from tests.testingutils import autospec_method from tron import mcp from tron.config import config_parse from tron.config import manager from tron.core.job_collection import JobCollection from tron.serialize.runstate import statemanager class TestMasterControlProgram: TEST_CONFIG = "tests/data/test_config.yaml" @pytest.fixture(autouse=True) def setup_mcp(self): self.working_dir = tempfile.mkdtemp() self.config_path = tempfile.mkdtemp() self.boot_time = time.time() self.mcp = mcp.MasterControlProgram(self.working_dir, self.config_path, self.boot_time) self.mcp.state_watcher = mock.create_autospec( statemanager.StateChangeWatcher, ) yield shutil.rmtree(self.config_path) shutil.rmtree(self.working_dir) def test_reconfigure_default(self): autospec_method(self.mcp._load_config) self.mcp.state_watcher = mock.MagicMock() self.mcp.reconfigure() self.mcp._load_config.assert_called_with(reconfigure=True, namespace_to_reconfigure=None) def test_reconfigure_namespace(self): autospec_method(self.mcp._load_config) self.mcp.state_watcher = mock.MagicMock() self.mcp.reconfigure(namespace="foo") self.mcp._load_config.assert_called_with(reconfigure=True, namespace_to_reconfigure="foo") @pytest.mark.parametrize( "reconfigure,namespace", [ (False, None), (True, None), (True, "foo"), ], ) def test_load_config(self, reconfigure, namespace): autospec_method(self.mcp.apply_config) self.mcp.config = mock.create_autospec(manager.ConfigManager) self.mcp._load_config(reconfigure, namespace) self.mcp.state_watcher.disabled.assert_called_with() self.mcp.apply_config.assert_called_with( self.mcp.config.load.return_value, reconfigure=reconfigure, namespace_to_reconfigure=namespace, ) @pytest.mark.parametrize( "reconfigure,namespace", [ (False, None), (True, None), (True, "foo"), (True, "MASTER"), ], ) @mock.patch("tron.mcp.KubernetesClusterRepository", autospec=True) @mock.patch("tron.mcp.MesosClusterRepository", autospec=True) @mock.patch("tron.mcp.node.NodePoolRepository", autospec=True) def test_apply_config(self, mock_repo, mock_cluster_repo, mock_k8s_cluster_repo, reconfigure, namespace): config_container = mock.create_autospec(config_parse.ConfigContainer) master_config = config_container.get_master.return_value autospec_method(self.mcp.jobs.update_from_config) autospec_method(self.mcp.build_job_scheduler_factory) self.mcp.apply_config(config_container, reconfigure, namespace) self.mcp.state_watcher.update_from_config.assert_called_with( master_config.state_persistence, ) assert_equal(self.mcp.context.base, master_config.command_context) mock_repo.update_from_config.assert_called_with( master_config.nodes, master_config.node_pools, master_config.ssh_options, ) mock_cluster_repo.configure.assert_called_with( master_config.mesos_options, ) mock_k8s_cluster_repo.configure.assert_called_with( master_config.k8s_options, ) self.mcp.build_job_scheduler_factory(master_config, mock.Mock()) expected_namespace_to_update = None if namespace == "MASTER" else namespace self.mcp.jobs.update_from_config.assert_called_once_with( config_container.get_jobs(), self.mcp.build_job_scheduler_factory.return_value, reconfigure, expected_namespace_to_update, ) self.mcp.state_watcher.watch_all.assert_called_once_with( self.mcp.jobs.update_from_config.return_value, mock.ANY, ) def test_update_state_watcher_config_changed(self): self.mcp.state_watcher.update_from_config.return_value = True self.mcp.jobs = mock.create_autospec(JobCollection) self.mcp.jobs.__iter__.return_values = { "a": mock.Mock(), "b": mock.Mock(), } state_config = mock.Mock() self.mcp.update_state_watcher_config(state_config) self.mcp.state_watcher.update_from_config.assert_called_with( state_config, ) assert_equal( self.mcp.state_watcher.save_job.mock_calls, [mock.call(j.job) for j in self.mcp.jobs], ) def test_update_state_watcher_config_no_change(self): self.mcp.state_watcher.update_from_config.return_value = False self.mcp.jobs = {"a": mock.Mock(), "b": mock.Mock()} state_config = mock.Mock() self.mcp.update_state_watcher_config(state_config) assert not self.mcp.state_watcher.save_job.mock_calls class TestMasterControlProgramRestoreState(TestCase): @setup def setup_mcp(self): self.working_dir = tempfile.mkdtemp() self.config_path = tempfile.mkdtemp() self.boot_time = time.time() self.mcp = mcp.MasterControlProgram(self.working_dir, self.config_path, self.boot_time) self.mcp.jobs = mock.create_autospec(JobCollection) self.mcp.state_watcher = mock.create_autospec( statemanager.StateChangeWatcher, ) @teardown def teardown_mcp(self): shutil.rmtree(self.working_dir) shutil.rmtree(self.config_path) def test_restore_state(self): job_state_data = {"1": "things", "2": "things"} state_data = { "job_state": job_state_data, } self.mcp.state_watcher.restore.return_value = state_data action_runner = mock.Mock() self.mcp.restore_state(action_runner) self.mcp.jobs.restore_state.assert_called_with(job_state_data, action_runner) if __name__ == "__main__": run() ================================================ FILE: tests/mesos_test.py ================================================ from collections import namedtuple from unittest import mock import staticconf.testing from testifycompat import assert_equal from testifycompat import setup_teardown from testifycompat import TestCase from tron.mesos import MesosCluster from tron.mesos import MesosClusterRepository from tron.mesos import MesosTask class TestMesosClusterRepository(TestCase): @setup_teardown def mock_cluster(self): # Ensure different mock is returned each time class is instantiated def init_cluster(*args, **kwargs): return mock.MagicMock(spec_set=MesosCluster) with mock.patch( "tron.mesos.MesosCluster", side_effect=init_cluster, autospec=True, ) as self.cluster_cls: yield def test_get_cluster_repeated_mesos_address(self): first = MesosClusterRepository.get_cluster("master-a.com") second = MesosClusterRepository.get_cluster("master-a.com") assert_equal(first, second) assert_equal(self.cluster_cls.call_count, 1) def test_shutdown(self): clusters = [MesosClusterRepository.get_cluster(address) for address in ["a", "b", "c"]] assert_equal(self.cluster_cls.call_count, 3) MesosClusterRepository.shutdown() for cluster in clusters: assert_equal(cluster.stop.call_count, 1) def test_configure(self): clusters = [MesosClusterRepository.get_cluster(address) for address in ["d", "e"]] mock_volume = mock.Mock() options = mock.Mock( master_port=5000, secret="/dev/null", principal="fake-principal", role="tron", enabled=False, default_volumes=[mock_volume], dockercfg_location="auth", offer_timeout=1000, ) with mock.patch( "tron.mesos.get_secret_from_file", autospec=True, return_value="test-secret", ): MesosClusterRepository.configure(options) expected_volume = mock_volume._asdict.return_value for cluster in clusters: cluster.set_enabled.assert_called_once_with(False) cluster.configure_tasks.assert_called_once_with( default_volumes=[expected_volume], dockercfg_location="auth", offer_timeout=1000, ) # Next cluster we get should be initialized with the same settings MesosClusterRepository.get_cluster("f") self.cluster_cls.assert_called_with( mesos_address="f", mesos_master_port=5000, secret="test-secret", principal="fake-principal", mesos_role="tron", framework_id=None, enabled=False, default_volumes=[expected_volume], dockercfg_location="auth", offer_timeout=1000, ) def mock_task_event( task_id, platform_type, raw=None, terminal=False, success=False, **kwargs, ): return mock.MagicMock( kind="task", task_id=task_id, platform_type=platform_type, raw=raw or {}, terminal=terminal, success=success, **kwargs, ) class TestMesosTask(TestCase): @setup_teardown def setup(self): TaskConfig = namedtuple("TaskConfig", "cmd task_id cpus mem disk env") self.action_run_id = "my_service.job.1.action" self.task_id = "123abcuuid" with mock.patch( "tron.mesos.logging.getLogger", return_value=mock.Mock(handlers=[mock.Mock()]), autospec=None, ): self.task = MesosTask( id=self.action_run_id, task_config=TaskConfig( cmd="echo hello world", task_id=self.task_id, cpus=0.1, mem=100, disk=100, env={ "INITIAL_VAR": "baz", "AWS_SECRET_ACCESS_KEY": "THISISASECRET", "SOME_VAR": "bar", "AWS_ACCESS_KEY_ID": "THISISASECRETTOO", "SOME_OTHER_VAR": "foo", }, ), ) yield def test_aws_credentials_redacted(self): assert all(["THISISASECRET" not in text[0][0] for text in self.task.log.info.call_args_list]) assert all(["foo" in text[0][0] for text in self.task.log.info.call_args_list]) assert all(["bar" in text[0][0] for text in self.task.log.info.call_args_list]) assert all(["baz" in text[0][0] for text in self.task.log.info.call_args_list]) def test_handle_staging(self): event = mock_task_event( task_id=self.task_id, platform_type="staging", ) self.task.handle_event(event) assert self.task.state == MesosTask.PENDING def test_handle_starting(self): event = mock_task_event( task_id=self.task_id, platform_type="starting", ) self.task.handle_event(event) assert self.task.state == MesosTask.RUNNING def test_handle_running(self): event = mock_task_event( task_id=self.task_id, platform_type="running", ) self.task.handle_event(event) assert self.task.state == MesosTask.RUNNING def test_handle_running_for_other_task(self): event = mock_task_event( task_id="other321", platform_type="running", ) self.task.handle_event(event) assert self.task.state == MesosTask.PENDING def test_handle_finished(self): self.task.started() event = mock_task_event( task_id=self.task_id, platform_type="finished", terminal=True, success=True, ) self.task.handle_event(event) assert self.task.is_complete def test_handle_failed(self): self.task.started() event = mock_task_event( task_id=self.task_id, platform_type="failed", terminal=True, success=False, ) self.task.handle_event(event) assert self.task.is_failed assert self.task.is_done def test_handle_killed(self): self.task.started() event = mock_task_event( task_id=self.task_id, platform_type="killed", terminal=True, success=False, ) self.task.handle_event(event) assert self.task.is_failed assert self.task.is_done def test_handle_lost(self): self.task.started() event = mock_task_event( task_id=self.task_id, platform_type="lost", terminal=True, success=False, ) self.task.handle_event(event) assert self.task.is_unknown assert self.task.is_done def test_handle_error(self): self.task.started() event = mock_task_event( task_id=self.task_id, platform_type="error", terminal=True, success=False, ) self.task.handle_event(event) assert self.task.is_failed assert self.task.is_done def test_handle_terminal_event_offer_timeout(self): self.task.started() event = mock_task_event( task_id=self.task_id, platform_type=None, terminal=True, success=False, raw="failed due to offer timeout", message="stop", ) self.task.handle_event(event) assert self.task.is_failed assert self.task.is_done def test_handle_success_sequence(self): self.task.handle_event( mock_task_event( task_id=self.task_id, platform_type="staging", ), ) self.task.handle_event( mock_task_event( task_id=self.task_id, platform_type="starting", ), ) self.task.handle_event( mock_task_event( task_id=self.task_id, platform_type="running", ), ) self.task.handle_event( mock_task_event( task_id=self.task_id, platform_type="finished", terminal=True, success=True, ), ) assert self.task.is_complete def test_log_event_error(self): with mock.patch.object(self.task, "log_event_info",) as mock_log_event, mock.patch.object( self.task.log, "warning", ) as mock_log: mock_log_event.side_effect = Exception self.task.handle_event( mock_task_event( task_id=self.task_id, platform_type="running", ), ) assert mock_log_event.called assert mock_log.called assert self.task.state == MesosTask.RUNNING def test_get_event_logger_add_unique_handlers(self): """ Ensures that only a single handler (for stderr) is added to the MesosTask event logger, to prevent duplicate log output. """ # Call 2 times to make sure 2nd call doesn't add another handler logger = self.task.get_event_logger() logger = self.task.get_event_logger() assert len(logger.handlers) == 1 class TestMesosCluster(TestCase): @setup_teardown def setup_mocks(self): with mock.patch("tron.mesos.PyDeferredQueue", autospec=True,) as queue_cls, mock.patch( "tron.mesos.TaskProcessor", autospec=True, ) as processor_cls, mock.patch("tron.mesos.Subscription", autospec=True,) as runner_cls, mock.patch( "tron.mesos.get_mesos_leader", autospec=True, ) as mock_get_leader: self.mock_queue = queue_cls.return_value self.mock_processor = processor_cls.return_value self.mock_runner_cls = runner_cls self.mock_runner_cls.return_value.configure_mock( stopping=False, TASK_CONFIG_INTERFACE=mock.Mock(), ) self.mock_get_leader = mock_get_leader yield @mock.patch("tron.mesos.socket", autospec=True) def test_init(self, mock_socket): mock_socket.gethostname.return_value = "hostname" cluster = MesosCluster( mesos_address="mesos-cluster-a.me", mesos_master_port=5000, secret="my_secret", mesos_role="tron", framework_id="fake_framework_id", principal="fake-principal", ) assert_equal(cluster.queue, self.mock_queue) assert_equal(cluster.processor, self.mock_processor) self.mock_get_leader.assert_called_once_with( "mesos-cluster-a.me", 5000, ) self.mock_processor.executor_from_config.assert_has_calls( [ mock.call( provider="mesos_task", provider_config={ "secret": "my_secret", "principal": "fake-principal", "mesos_address": self.mock_get_leader.return_value, "role": "tron", "framework_name": "tron-hostname", "framework_id": "fake_framework_id", "failover": True, }, ), mock.call( provider="logging", provider_config=mock.ANY, ), ] ) self.mock_runner_cls.assert_called_once_with( self.mock_processor.executor_from_config.return_value, self.mock_queue, ) assert_equal(cluster.runner, self.mock_runner_cls.return_value) get_event_deferred = cluster.deferred assert_equal(get_event_deferred, self.mock_queue.get.return_value) get_event_deferred.addCallback.assert_has_calls( [ mock.call(cluster._process_event), mock.call(cluster.handle_next_event), ] ) def test_init_disabled(self): cluster = MesosCluster("mesos-cluster-a.me", enabled=False) assert_equal(cluster.queue, self.mock_queue) assert_equal(cluster.processor, self.mock_processor) assert_equal(self.mock_processor.executor_from_config.call_count, 0) assert cluster.runner is None def test_set_enabled_off(self): cluster = MesosCluster("mesos-cluster-a.me", enabled=True) mock_task = mock.Mock() cluster.tasks = {"task": mock_task} cluster.set_enabled(False) assert not cluster.enabled assert cluster.runner.stop.call_count == 1 assert cluster.tasks == {} assert mock_task.exited.call_count == 1 def test_set_enabled_on(self): cluster = MesosCluster("mesos-cluster-a.me", enabled=False) cluster.set_enabled(True) assert_equal(cluster.enabled, True) # Basically the same as regular initialization assert_equal(self.mock_processor.executor_from_config.call_count, 2) self.mock_runner_cls.assert_called_once_with( self.mock_processor.executor_from_config.return_value, self.mock_queue, ) assert_equal(cluster.runner, self.mock_runner_cls.return_value) get_event_deferred = cluster.deferred assert_equal(get_event_deferred, self.mock_queue.get.return_value) get_event_deferred.addCallback.assert_has_calls( [ mock.call(cluster._process_event), mock.call(cluster.handle_next_event), ] ) def test_set_enabled_on_already(self): cluster = MesosCluster("mesos-cluster-a.me", enabled=True) cluster.set_enabled(True) assert_equal(cluster.enabled, True) # Runner should have only be created once assert_equal(self.mock_runner_cls.call_count, 1) def test_configure_tasks(self): cluster = MesosCluster( "mesos-cluster-a.me", default_volumes=[], dockercfg_location="first", offer_timeout=60, ) assert_equal(cluster.default_volumes, []) assert_equal(cluster.dockercfg_location, "first") assert_equal(cluster.offer_timeout, 60) expected_volumes = [ { "container_path": "/tmp", "host_path": "/host", "mode": "RO", } ] cluster.configure_tasks( default_volumes=expected_volumes, dockercfg_location="second", offer_timeout=300, ) assert_equal(cluster.default_volumes, expected_volumes) assert_equal(cluster.dockercfg_location, "second") assert_equal(cluster.offer_timeout, 300) def test_submit(self): mock_clusterman_metrics = mock.MagicMock() cluster = MesosCluster("mesos-cluster-a.me") mock_task = mock.MagicMock(get_config=mock.Mock(return_value={"environment": {}})) mock_task.get_mesos_id.return_value = "this_task" with mock.patch( "tron.mesos.get_clusterman_metrics", return_value=(mock_clusterman_metrics), autospec=True, ): cluster.submit(mock_task) assert "this_task" in cluster.tasks assert cluster.tasks["this_task"] == mock_task cluster.runner.run.assert_called_once_with( mock_task.get_config.return_value, ) assert mock_clusterman_metrics.ClustermanMetricsBotoClient.call_count == 0 def test_submit_with_clusterman(self): mock_clusterman_metrics = mock.MagicMock() cluster = MesosCluster("mesos-cluster-a.me") mock_task = mock.MagicMock( get_config=mock.Mock( return_value={ "environment": { "CLUSTERMAN_RESOURCES": '{"required_cpus|blah=x": 4}', "EXECUTOR_CLUSTER": "fake-cluster", "EXECUTOR_POOL": "fake-pool", }, }, ), ) mock_task.get_mesos_id.return_value = "this_task" with mock.patch( "tron.mesos.get_clusterman_metrics", return_value=mock_clusterman_metrics, autospec=True, ), staticconf.testing.MockConfiguration( {"clusters": {"fake-cluster": {"aws_region": "fake-region"}}}, namespace="clusterman", ): cluster.submit(mock_task) assert "this_task" in cluster.tasks assert cluster.tasks["this_task"] == mock_task cluster.runner.run.assert_called_once_with( mock_task.get_config.return_value, ) assert mock_clusterman_metrics.ClustermanMetricsBotoClient.call_count == 1 def test_submit_disabled(self): cluster = MesosCluster("mesos-cluster-a.me", enabled=False) mock_task = mock.MagicMock() mock_task.get_mesos_id.return_value = "this_task" with mock.patch( "tron.mesos.get_clusterman_metrics", return_value=(None, None), autospec=True, ): cluster.submit(mock_task) assert "this_task" not in cluster.tasks mock_task.exited.assert_called_once_with(1) def test_recover(self): cluster = MesosCluster("mesos-cluster-a.me") mock_task = mock.MagicMock() mock_task.get_mesos_id.return_value = "this_task" cluster.recover(mock_task) assert "this_task" in cluster.tasks assert cluster.tasks["this_task"] == mock_task cluster.runner.reconcile.assert_called_once_with( mock_task.get_config.return_value, ) assert mock_task.started.call_count == 1 def test_recover_disabled(self): cluster = MesosCluster("mesos-cluster-a.me", enabled=False) mock_task = mock.MagicMock() mock_task.get_mesos_id.return_value = "this_task" cluster.recover(mock_task) assert "this_task" not in cluster.tasks mock_task.exited.assert_called_once_with(None) @mock.patch("tron.mesos.MesosTask", autospec=True) def test_create_task_defaults(self, mock_task): cluster = MesosCluster("mesos-cluster-a.me") mock_serializer = mock.MagicMock() task = cluster.create_task( action_run_id="action_c", command="echo hi", cpus=1, mem=10, disk=20, constraints=[], docker_image="container:latest", docker_parameters=[], env={"TESTING": "true"}, extra_volumes=[], serializer=mock_serializer, ) cluster.runner.TASK_CONFIG_INTERFACE.assert_called_once_with( name="action_c", cmd="echo hi", cpus=1, mem=10, disk=20, constraints=[], image="container:latest", docker_parameters=[], environment={"TESTING": "true"}, volumes=[], uris=[], offer_timeout=None, ) assert_equal(task, mock_task.return_value) mock_task.assert_called_once_with( "action_c", cluster.runner.TASK_CONFIG_INTERFACE.return_value, mock_serializer, ) @mock.patch("tron.mesos.MesosTask", autospec=True) def test_create_task_with_task_id(self, mock_task): cluster = MesosCluster("mesos-cluster-a.me") mock_serializer = mock.MagicMock() task_id = "task.0123-fabc" task = cluster.create_task( action_run_id="action_c", command="echo hi", cpus=1, mem=10, disk=20, constraints=[], docker_image="container:latest", docker_parameters=[], env={"TESTING": "true"}, extra_volumes=[], serializer=mock_serializer, task_id=task_id, ) assert cluster.runner.TASK_CONFIG_INTERFACE.call_count == 1 assert task == mock_task.return_value task_config = cluster.runner.TASK_CONFIG_INTERFACE.return_value task_config.set_task_id.assert_called_once_with(task_id) mock_task.assert_called_once_with( "action_c", task_config.set_task_id.return_value, mock_serializer, ) @mock.patch("tron.mesos.MesosTask", autospec=True) def test_create_task_disabled(self, mock_task): # If Mesos is disabled, should return None cluster = MesosCluster("mesos-cluster-a.me", enabled=False) mock_serializer = mock.MagicMock() task = cluster.create_task( action_run_id="action_c", command="echo hi", cpus=1, mem=10, disk=20, constraints=[], docker_image="container:latest", docker_parameters=[], env={"TESTING": "true"}, extra_volumes=[], serializer=mock_serializer, ) assert task is None @mock.patch("tron.mesos.MesosTask", autospec=True) def test_create_task_with_configuration(self, mock_task): cluster = MesosCluster( "mesos-cluster-a.me", default_volumes=[ { "container_path": "/tmp", "host_path": "/host", "mode": "RO", }, { "container_path": "/other", "host_path": "/other", "mode": "RW", }, ], dockercfg_location="some_place", offer_timeout=202, ) mock_serializer = mock.MagicMock() task = cluster.create_task( action_run_id="action_c", command="echo hi", cpus=1, mem=10, disk=20, constraints=[], docker_image="container:latest", docker_parameters=[], env={"TESTING": "true"}, # This should override the default volume for /tmp extra_volumes=[ { "container_path": "/tmp", "host_path": "/custom", "mode": "RW", }, ], serializer=mock_serializer, ) cluster.runner.TASK_CONFIG_INTERFACE.assert_called_once_with( name="action_c", cmd="echo hi", cpus=1, mem=10, disk=20, constraints=[], image="container:latest", docker_parameters=[], environment={"TESTING": "true"}, volumes=[ { "container_path": "/tmp", "host_path": "/custom", "mode": "RW", }, { "container_path": "/other", "host_path": "/other", "mode": "RW", }, ], uris=["some_place"], offer_timeout=202, ) assert_equal(task, mock_task.return_value) mock_task.assert_called_once_with( "action_c", cluster.runner.TASK_CONFIG_INTERFACE.return_value, mock_serializer, ) def test_process_event_task(self): event = mock_task_event("this_task", "some_platform_type") cluster = MesosCluster("mesos-cluster-a.me") mock_task = mock.MagicMock(spec_set=MesosTask) mock_task.get_mesos_id.return_value = "this_task" cluster.tasks["this_task"] = mock_task cluster._process_event(event) mock_task.handle_event.assert_called_once_with(event) def test_process_event_task_id_invalid(self): event = mock_task_event("other_task", "some_platform_type") cluster = MesosCluster("mesos-cluster-a.me") mock_task = mock.MagicMock(spec_set=MesosTask) mock_task.get_mesos_id.return_value = "this_task" cluster.tasks["this_task"] = mock_task cluster._process_event(event) assert_equal(mock_task.handle_event.call_count, 0) def test_process_event_control_stop(self): event = mock.MagicMock( kind="control", message="stop", ) cluster = MesosCluster("mesos-cluster-a.me") cluster._process_event(event) assert cluster.runner.stop.call_count == 1 assert cluster.deferred is None def test_stop_default(self): # When stopping, tasks should not exit. They will be recovered cluster = MesosCluster("mesos-cluster-a.me") mock_task = mock.MagicMock() cluster.tasks = {"task_id": mock_task} cluster.stop() assert cluster.runner.stop.call_count == 1 assert cluster.deferred is None assert mock_task.exited.call_count == 0 assert len(cluster.tasks) == 1 def test_stop_disabled(self): # Shouldn't raise an error cluster = MesosCluster("mesos-cluster-a.me", enabled=False) cluster.stop() def test_kill(self): cluster = MesosCluster("mesos-cluster-a.me") cluster.kill("fake_task_id") cluster.runner.kill.assert_called_once_with("fake_task_id") ================================================ FILE: tests/metrics_test.py ================================================ from unittest import mock import pytest import tron.metrics as metrics @pytest.fixture(autouse=True) def all_metrics(): with mock.patch.object(metrics, "all_metrics", new=dict()) as mock_all: yield mock_all def test_get_metric(all_metrics): timer = metrics.get_metric( "timer", "api.requests", {"method": "GET"}, mock.Mock(), ) same_timer = metrics.get_metric( "timer", "api.requests", {"method": "GET"}, mock.Mock(), ) other_timer = metrics.get_metric( "timer", "api.requests", {"method": "POST"}, mock.Mock(), ) metrics.get_metric("something", "name", None, mock.Mock()) assert timer == same_timer assert other_timer != timer assert len(all_metrics) == 3 @mock.patch("tron.metrics.get_metric", autospec=True) def test_timer(mock_get_metric): test_metric = metrics.Timer() mock_get_metric.return_value = test_metric metrics.timer("my_metric", 110) metrics.timer("my_metric", 84) mock_get_metric.assert_called_with( "timer", "my_metric", None, mock.ANY, ) result = metrics.view_timer(test_metric) assert result["count"] == 2 @mock.patch("tron.metrics.get_metric", autospec=True) def test_count(mock_get_metric): test_metric = metrics.Counter() mock_get_metric.return_value = test_metric metrics.count("my_metric", 13) metrics.count("my_metric", -1) mock_get_metric.assert_called_with( "counter", "my_metric", None, mock.ANY, ) result = metrics.view_counter(test_metric) assert result["count"] == 12 @mock.patch("tron.metrics.get_metric", autospec=True) def test_meter(mock_get_metric): test_metric = metrics.Meter() mock_get_metric.return_value = test_metric metrics.meter("my_metric") metrics.meter("my_metric") mock_get_metric.assert_called_with( "meter", "my_metric", None, mock.ANY, ) result = metrics.view_meter(test_metric) assert result["count"] == 2 @mock.patch("tron.metrics.get_metric", autospec=True) def test_gauge(mock_get_metric): test_metric = metrics.SimpleGauge() mock_get_metric.return_value = test_metric metrics.gauge("my_metric", 23) metrics.gauge("my_metric", 102) mock_get_metric.assert_called_with( "gauge", "my_metric", None, mock.ANY, ) result = metrics.view_gauge(test_metric) assert result["value"] == 102 @mock.patch("tron.metrics.get_metric", autospec=True) def test_histogram(mock_get_metric): test_metric = metrics.Histogram() mock_get_metric.return_value = test_metric metrics.histogram("my_metric", 2) metrics.histogram("my_metric", 92) mock_get_metric.assert_called_with( "histogram", "my_metric", None, mock.ANY, ) result = metrics.view_histogram(test_metric) assert result["count"] == 2 def test_view_all_metrics_empty(): result = metrics.view_all_metrics() assert result == { "counter": [], "gauge": [], "histogram": [], "meter": [], "timer": [], } def test_view_all_metrics(): metrics.timer("a", 1) metrics.count("b", 9, dimensions={"method": "GET"}) metrics.meter("c") metrics.gauge("d", 3) metrics.histogram("e", 2) metrics.histogram("f", 3) result = metrics.view_all_metrics() assert len(result["timer"]) == 1 assert result["timer"][0]["name"] == "a" assert len(result["counter"]) == 1 assert result["counter"][0]["name"] == "b" assert result["counter"][0]["dimensions"] == {"method": "GET"} assert len(result["meter"]) == 1 assert result["meter"][0]["name"] == "c" assert len(result["gauge"]) == 1 assert result["gauge"][0]["name"] == "d" assert len(result["histogram"]) == 2 names = {metric["name"] for metric in result["histogram"]} assert names == {"e", "f"} ================================================ FILE: tests/mocks.py ================================================ import atexit import datetime import itertools import shutil import tempfile from unittest.mock import MagicMock class MockAction(MagicMock): def __init__(self, *args, **kwargs): kwargs.setdefault("name", "action_name") kwargs.setdefault("required_actions", []) kwargs.setdefault("dependent_actions", []) super().__init__(*args, **kwargs) class MockActionGraph(MagicMock): def __init__(self, *args, **kwargs): action = MockAction() kwargs.setdefault("graph", [action]) kwargs.setdefault("action_map", {action.name: action}) super().__init__(*args, **kwargs) def __getitem__(self, item): action = MockAction(name=item) self.action_map.setdefault(item, action) return self.action_map[item] def get_required_actions(self, name): return [] class MockActionRun(MagicMock): def __init__(self, *args, **kwargs): kwargs.setdefault("output_path", [tempfile.mkdtemp()]) kwargs.setdefault("start_time", datetime.datetime.now()) kwargs.setdefault("end_time", datetime.datetime.now()) atexit.register(lambda: shutil.rmtree(kwargs["output_path"][0])) super().__init__(*args, **kwargs) class MockActionRunCollection(MagicMock): def __init__(self, *args, **kwargs): kwargs.setdefault("action_graph", MockActionGraph()) kwargs.setdefault("run_map", {}) super().__init__(*args, **kwargs) def __getitem__(self, item): action_run = MockActionRun(name=item) self.run_map.setdefault(item, action_run) return self.run_map[item] class MockJobRun(MagicMock): def __init__(self, *args, **kwargs): kwargs.setdefault("output_path", [tempfile.mkdtemp()]) kwargs.setdefault("action_graph", MockActionGraph()) action_runs = MockActionRunCollection( action_graph=kwargs["action_graph"], ) kwargs.setdefault("action_runs", action_runs) atexit.register(lambda: shutil.rmtree(kwargs["output_path"][0])) super().__init__(*args, **kwargs) class MockNode(MagicMock): def __init__(self, hostname=None): super().__init__() self.name = self.hostname = hostname def run(self, runnable): runnable.started() return type(self)() class MockNodePool: _node = None def __init__(self, *node_names): self.nodes = [] self._ndx_cycle = None for hostname in node_names: self.nodes.append(MockNode(hostname=hostname)) if self.nodes: self._ndx_cycle = itertools.cycle(range(0, len(self.nodes))) def __getitem__(self, value): for node in self.nodes: if node.hostname == value: return node else: raise KeyError def next(self): if not self.nodes: self.nodes.append(MockNode()) if self._ndx_cycle: return self.nodes[next(self._ndx_cycle)] else: return self.nodes[0] next_round_robin = next class MockJobRunCollection(MagicMock): def __iter__(self): return iter(self.runs) ================================================ FILE: tests/node_test.py ================================================ from unittest import mock from testifycompat import assert_equal from testifycompat import assert_in from testifycompat import assert_not_equal from testifycompat import assert_not_in from testifycompat import assert_raises from testifycompat import run from testifycompat import setup from testifycompat import setup_teardown from testifycompat import teardown from testifycompat import TestCase from tests.testingutils import autospec_method from tron import actioncommand from tron import node from tron import ssh from tron.config import schema from tron.core import actionrun from tron.serialize import filehandler def create_mock_node(name=None): mock_node = mock.create_autospec(node.Node) if name: mock_node.get_name.return_value = name return mock_node def create_mock_pool(): return mock.create_autospec(node.NodePool) class TestNodePoolRepository(TestCase): @setup def setup_store(self): self.node = create_mock_node() self.repo = node.NodePoolRepository.get_instance() self.repo.add_node(self.node) @teardown def teardown_store(self): self.repo.clear() def test_single_instance(self): assert_raises(ValueError, node.NodePoolRepository) assert self.repo is node.NodePoolRepository.get_instance() def test_get_by_name(self): node_pool = self.repo.get_by_name(self.node.get_name()) assert_equal(self.node, node_pool.next()) def test_get_by_name_miss(self): assert_equal(None, self.repo.get_by_name("bogus")) def test_clear(self): self.repo.clear() assert_not_in(self.node, self.repo.nodes) assert_not_in(self.node, self.repo.pools) def test_update_from_config(self): mock_nodes = {"a": create_mock_node("a"), "b": create_mock_node("b")} self.repo.nodes.update(mock_nodes) node_config = {"a": mock.Mock(), "b": mock.Mock()} node_pool_config = {"c": mock.Mock(nodes=["a", "b"])} ssh_options = mock.Mock(identities=[], known_hosts_file=None) node.NodePoolRepository.update_from_config( node_config, node_pool_config, ssh_options, ) node_names = [node_config["a"].name, node_config["b"].name] assert_equal( set(self.repo.pools), set(node_names + [node_pool_config["c"].name]), ) assert_equal( set(self.repo.nodes), set(list(node_names) + list(mock_nodes.keys())), ) def test_nodes_by_name(self): mock_nodes = {"a": mock.Mock(), "b": mock.Mock()} self.repo.nodes.update(mock_nodes) nodes = self.repo._get_nodes_by_name(["a", "b"]) assert_equal(nodes, list(mock_nodes.values())) def test_get_node(self): returned_node = self.repo.get_node(self.node.get_name()) assert_equal(returned_node, self.node) class TestKnownHost(TestCase): @setup def setup_known_hosts(self): self.known_hosts = node.KnownHosts(None) self.entry = mock.Mock() self.known_hosts._added.append(self.entry) def test_get_public_key(self): hostname = "hostname" pub_key = self.known_hosts.get_public_key(hostname) self.entry.matchesHost.assert_called_with(hostname) assert_equal(pub_key, self.entry.publicKey) def test_get_public_key_not_found(self): self.entry.matchesHost.return_value = False assert not self.known_hosts.get_public_key("hostname") class TestDetermineJitter(TestCase): @setup def setup_node_settings(self): self.settings = mock.Mock( jitter_load_factor=1, jitter_min_load=4, jitter_max_delay=20, ) @setup_teardown def patch_random(self): with mock.patch("tron.node.random", autospec=True) as mock_random: mock_random.random.return_value = 1 yield def test_jitter_under_min_load(self): assert_equal(node.determine_jitter(3, self.settings), 0) assert_equal(node.determine_jitter(4, self.settings), 0) def test_jitter_with_load_factor(self): self.settings.jitter_load_factor = 2 assert_equal(node.determine_jitter(3, self.settings), 2.0) assert_equal(node.determine_jitter(2, self.settings), 0) def test_jitter_with_max_delay(self): self.settings.jitter_max_delay = 15 assert_equal(node.determine_jitter(20, self.settings), 15.0) assert_equal(node.determine_jitter(100, self.settings), 15.0) def build_node( hostname="localhost", username="theuser", name="thename", pub_key=None, ): config = mock.Mock(hostname=hostname, username=username, name=name) ssh_opts = mock.create_autospec(ssh.SSHAuthOptions) node_settings = mock.create_autospec(schema.ConfigSSHOptions) return node.Node(config, ssh_opts, pub_key, node_settings) class TestNode(TestCase): class TestConnection: def openChannel(self, chan): self.chan = chan @setup def setup_node(self): self.node = build_node() def test_output_logging(self): test_node = build_node() serializer = mock.create_autospec(filehandler.FileHandleManager) action_cmd = actionrun.ActionCommand("test", "false", serializer) test_node.connection = self.TestConnection() test_node.run_states = {action_cmd.id: mock.Mock(state=0)} test_node.run_states[action_cmd.id].state = node.RUN_STATE_CONNECTING test_node.run_states[action_cmd.id].run = action_cmd test_node._open_channel(action_cmd) assert test_node.connection.chan is not None test_node.connection.chan.dataReceived("test") serializer.open.return_value.write.assert_called_with("test") def test_from_config(self): ssh_options = self.node.conch_options node_config = mock.Mock( hostname="localhost", username="theuser", name="thename", ) ssh_options.__getitem__.return_value = "something" public_key = mock.Mock() node_settings = mock.Mock() new_node = node.Node.from_config( node_config, ssh_options, public_key, node_settings, ) assert_equal(new_node.name, node_config.name) assert_equal(new_node.hostname, node_config.hostname) assert_equal(new_node.username, node_config.username) assert_equal(new_node.pub_key, public_key) assert_equal(new_node.node_settings, node_settings) def test__eq__true(self): other_node = build_node() other_node.conch_options = self.node.conch_options other_node.node_settings = self.node.node_settings other_node.config = self.node.config assert_equal(other_node, self.node) def test__eq__false_config_changed(self): other_node = build_node(username="different") assert_not_equal(other_node, self.node) def test__eq__false_pub_key_changed(self): other_node = build_node(pub_key="something") assert_not_equal(other_node, self.node) def test__eq__false_ssh_options_changed(self): other_node = build_node() other_node.conch_options = mock.create_autospec(ssh.SSHAuthOptions) assert_not_equal(other_node, self.node) def test_stop_not_tracked(self): action_command = mock.create_autospec( actioncommand.ActionCommand, id=mock.Mock(), ) self.node.stop(action_command) def test_stop(self): autospec_method(self.node._fail_run) action_command = mock.create_autospec( actioncommand.ActionCommand, id=mock.Mock(), ) self.node.run_states[action_command.id] = mock.Mock() self.node.stop(action_command) assert_equal(self.node._fail_run.call_count, 1) class TestNodePool(TestCase): @setup def setup_nodes(self): self.nodes = [build_node(name="node%s" % i) for i in range(5)] self.node_pool = node.NodePool(self.nodes, "thename") def test_from_config(self): name = "the pool name" nodes = [create_mock_node(), create_mock_node()] config = mock.Mock(name=name) new_pool = node.NodePool.from_config(config, nodes) assert_equal(new_pool.name, config.name) assert_equal(new_pool.nodes, nodes) def test__init__(self): new_node = node.NodePool(self.nodes, "thename") assert_equal(new_node.name, "thename") def test__eq__(self): other_pool = node.NodePool(self.nodes, "othername") assert_equal(self.node_pool, other_pool) def test_next(self): # Call next many times for _ in range(len(self.nodes) * 2 + 1): assert_in(self.node_pool.next(), self.nodes) def test_next_round_robin(self): node_order = [self.node_pool.next_round_robin() for _ in range(len(self.nodes) * 2)] assert_equal(node_order, self.nodes + self.nodes) if __name__ == "__main__": run() ================================================ FILE: tests/sandbox.py ================================================ import contextlib import functools import logging import os import shutil import signal import socket import sys import tempfile import time from subprocess import CalledProcessError from subprocess import PIPE from subprocess import Popen from unittest import mock from testifycompat import assert_not_equal from testifycompat import setup from testifycompat import teardown from testifycompat import TestCase from tron.commands import client from tron.config import manager from tron.config import schema # Used for getting the locations of the executable test_dir, _ = os.path.split(__file__) repo_root, _ = os.path.split(test_dir) log = logging.getLogger(__name__) def wait_on_sandbox(func, delay=0.1, max_wait=5.0): """Poll for func() to return True. Sleeps `delay` seconds between polls up to a max of `max_wait` seconds. """ start_time = time.time() while time.time() - start_time < max_wait: time.sleep(delay) if func(): return raise TronSandboxException("Failed %s" % func.__name__) def wait_on_state(client_func, url, state, field="state"): """Use client_func(url) to wait until the resource changes to state.""" def wait_func(): return client_func(url)[field] == state wait_func.__name__ = f"{url} wait on {state}" wait_on_sandbox(wait_func) def wait_on_proc_terminate(pid): def wait_on_terminate(): try: os.kill(pid, 0) except Exception: return True wait_on_terminate.__name__ = "Wait on %s to terminate" % pid wait_on_sandbox(wait_on_terminate) def build_waiter_func(client_func, url): return functools.partial(wait_on_state, client_func, url) def handle_output(cmd, out_err, returncode): """Log process output before it is parsed. Raise exception if exit code is nonzero. """ stdout, stderr = out_err cmd = " ".join(cmd) if stdout: log.warn("%s STDOUT: %s", cmd, stdout) if stderr: log.warn("%s STDERR: %s", cmd, stderr) if returncode: raise CalledProcessError(returncode, cmd) def find_unused_port(): """Return a port number that is not in use.""" sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) with contextlib.closing(sock) as sock: sock.bind(("localhost", 0)) _, port = sock.getsockname() return port class TronSandboxException(Exception): pass class SandboxTestCase(TestCase): _suites = ["sandbox"] sandbox = None @setup def make_sandbox(self): verify_environment() self.sandbox = TronSandbox() self.client = self.sandbox.client @teardown def delete_sandbox(self): if self.sandbox: self.sandbox.delete() self.sandbox = None def start_with_config(self, config): self.sandbox.save_config(config) self.sandbox.trond() def restart_trond(self): old_pid = self.sandbox.get_trond_pid() self.sandbox.shutdown_trond() wait_on_proc_terminate(self.sandbox.get_trond_pid()) self.sandbox.trond() assert_not_equal(old_pid, self.sandbox.get_trond_pid()) class ClientProxy: """Wrap calls to client and raise a TronSandboxException on connection failures. """ def __init__(self, client, log_filename): self.client = client self.log_filename = log_filename def log_contents(self): """Return the contents of the log file.""" with open(self.log_filename) as f: return f.read() def wrap(self, func, *args, **kwargs): with mock.patch("tron.commands.client.log", autospec=True): try: return func(*args, **kwargs) except (client.RequestError, ValueError) as e: # ValueError for JSONDecode errors log_contents = self.log_contents() if log_contents: log.warn(f"{e!r}, Log:\n{log_contents}") return False def __getattr__(self, name): attr = getattr(self.client, name) if not callable(attr): return attr return functools.partial(self.wrap, attr) def verify_environment(): for env_var in ["SSH_AUTH_SOCK", "PYTHONPATH"]: if not os.environ.get(env_var): raise TronSandboxException( "Missing $%s in test environment." % env_var, ) class TronSandbox: """A sandbox for running trond and tron commands in subprocesses.""" def __init__(self): """Set up a temp directory and store paths to relevant binaries""" self.tmp_dir = tempfile.mkdtemp(prefix="tron-") cmd_path_func = functools.partial(os.path.join, repo_root, "bin") cmds = "tronctl", "trond", "tronfig", "tronview" self.commands = {cmd: cmd_path_func(cmd) for cmd in cmds} self.log_file = self.abs_path("tron.log") self.log_conf = self.abs_path("logging.conf") self.pid_file = self.abs_path("tron.pid") self.config_path = self.abs_path("configs/") self.port = find_unused_port() self.host = "localhost" self.api_uri = f"http://{self.host}:{self.port}" cclient = client.Client(self.api_uri) self.client = ClientProxy(cclient, self.log_file) self.setup_logging_conf() def abs_path(self, filename): """Return the absolute path for a file in the sandbox.""" return os.path.join(self.tmp_dir, filename) def setup_logging_conf(self): config_template = os.path.join(repo_root, "tests/data/logging.conf") with open(config_template) as fh: config = fh.read() with open(self.log_conf, "w") as fh: fh.write(config.format(self.log_file)) def delete(self): """Delete the temp directory and shutdown trond.""" self.shutdown_trond(sig_num=signal.SIGKILL) shutil.rmtree(self.tmp_dir) def save_config(self, config_text): """Save the initial tron configuration.""" manager.create_new_config(self.config_path, config_text) def run_command(self, command_name, args=None, stdin_lines=None): """Run the command by name and return (stdout, stderr).""" args = args or [] command = [sys.executable, self.commands[command_name]] + args stdin = PIPE if stdin_lines else None proc = Popen(command, stdout=PIPE, stderr=PIPE, stdin=stdin) streams = proc.communicate(stdin_lines) try: handle_output(command, streams, proc.returncode) except CalledProcessError: log.warn(self.client.log_contents()) raise return streams def tronctl(self, *args): args = list(args) if args else [] return self.run_command("tronctl", args + ["--server", self.api_uri]) def tronview(self, *args): args = list(args) if args else [] args += ["--nocolor", "--server", self.api_uri] return self.run_command("tronview", args) def trond(self, *args): args = list(args) if args else [] args += [ "--working-dir=%s" % self.tmp_dir, "--pid-file=%s" % self.pid_file, "--port=%d" % self.port, "--host=%s" % self.host, "--config-path=%s" % self.config_path, "--log-conf=%s" % self.log_conf, ] self.run_command("trond", args) wait_on_sandbox(lambda: bool(self.client.home())) def tronfig( self, config_content=None, name=schema.MASTER_NAMESPACE, ): args = ["--server", self.api_uri, name] args += ["-"] if config_content else ["-p"] return self.run_command("tronfig", args, stdin_lines=config_content) def get_trond_pid(self): if not os.path.exists(self.pid_file): return None with open(self.pid_file) as f: return int(f.read()) def shutdown_trond(self, sig_num=signal.SIGTERM): trond_pid = self.get_trond_pid() if trond_pid: os.kill(trond_pid, sig_num) ================================================ FILE: tests/scheduler_test.py ================================================ import calendar import datetime from unittest import mock import pytz from testifycompat import assert_equal from testifycompat import assert_gt from testifycompat import assert_gte from testifycompat import assert_lt from testifycompat import assert_lte from testifycompat import run from testifycompat import setup from testifycompat import TestCase from tests import testingutils from tron import scheduler from tron.config import config_utils from tron.config import schedule_parse from tron.config.config_utils import NullConfigContext from tron.config.schedule_parse import parse_groc_expression from tron.utils import timeutils class TestSchedulerFromConfig(TestCase): def test_cron_scheduler(self): line = "cron */5 * * 7,8 *" config_context = mock.Mock(path="test") config = schedule_parse.valid_schedule(line, config_context) sched = scheduler.scheduler_from_config(config=config, time_zone=None) start_time = datetime.datetime(2012, 3, 14, 15, 9, 26) next_time = sched.next_run_time(start_time) assert_equal(next_time, datetime.datetime(2012, 7, 1, 0)) assert_equal(str(sched), "cron */5 * * 7,8 *") def test_daily_scheduler(self): config_context = config_utils.NullConfigContext line = "daily 17:32 MWF" config = schedule_parse.valid_schedule(line, config_context) sched = scheduler.scheduler_from_config(config=config, time_zone=None) assert_equal(sched.time_spec.hours, [17]) assert_equal(sched.time_spec.minutes, [32]) start_time = datetime.datetime(2012, 3, 14, 15, 9, 26) for day in [14, 16, 19]: next_time = sched.next_run_time(start_time) assert_equal(next_time, datetime.datetime(2012, 3, day, 17, 32)) start_time = next_time assert_equal(str(sched), "daily 17:32 MWF") class GeneralSchedulerTestCase(testingutils.MockTimeTestCase): now = datetime.datetime.now().replace(hour=15, minute=0) def expected_time(self, date): return datetime.datetime.combine(date, datetime.time(14, 30)) @setup def build_scheduler(self): self.scheduler = scheduler.GeneralScheduler(timestr="14:30") one_day = datetime.timedelta(days=1) self.today = self.now.date() self.yesterday = self.now - one_day self.tomorrow = self.now + one_day def test_next_run_time(self): next_run = self.scheduler.next_run_time(timeutils.current_time()) assert_equal(self.expected_time(self.tomorrow), next_run) next_run = self.scheduler.next_run_time(self.yesterday) assert_equal(self.expected_time(self.today), next_run) @mock.patch("tron.scheduler.get_jitter", autospec=True) def test_next_run_time_with_jitter(self, mock_jitter): mock_jitter.return_value = delta = datetime.timedelta(seconds=-300) self.scheduler.jitter = datetime.timedelta(seconds=400) expected = self.expected_time(self.tomorrow) + delta next_run_time = self.scheduler.next_run_time(None) assert_equal(next_run_time, expected) def test__str__(self): assert_equal(str(self.scheduler), "daily ") def test__str__with_jitter(self): self.scheduler.jitter = datetime.timedelta(seconds=300) assert_equal(str(self.scheduler), "daily (+/- 0:05:00)") class GeneralSchedulerTimeTestBase(testingutils.MockTimeTestCase): now = datetime.datetime(2012, 3, 14, 15, 9, 26) @setup def build_scheduler(self): self.scheduler = scheduler.GeneralScheduler(timestr="14:30") class GeneralSchedulerTodayTest(GeneralSchedulerTimeTestBase): now = datetime.datetime.now().replace(hour=12, minute=0) def test(self): # If we schedule a job for later today, it should run today run_time = self.scheduler.next_run_time(self.now) next_run_date = run_time.date() assert_equal(next_run_date, self.now.date()) earlier_time = datetime.datetime( self.now.year, self.now.month, self.now.day, hour=13, ) assert_lte(earlier_time, run_time) class GeneralSchedulerTomorrowTest(GeneralSchedulerTimeTestBase): now = datetime.datetime.now().replace(hour=15, minute=0) def test(self): # If we schedule a job for later today, it should run today run_time = self.scheduler.next_run_time(self.now) next_run_date = run_time.date() tomorrow = self.now.date() + datetime.timedelta(days=1) assert_equal(next_run_date, tomorrow) earlier_time = datetime.datetime( year=tomorrow.year, month=tomorrow.month, day=tomorrow.day, hour=13, ) assert_lte(earlier_time, run_time) class GeneralSchedulerLongJobRunTest(GeneralSchedulerTimeTestBase): now = datetime.datetime.now().replace(hour=12, minute=0) def test_long_jobs_dont_wedge_scheduler(self): # Advance days twice as fast as they are scheduled, demonstrating # that the scheduler will put things in the past if that's where # they belong, and run them as fast as possible last_run = self.scheduler.next_run_time(None) for i in range(10): next_run = self.scheduler.next_run_time(last_run) assert_equal(next_run, last_run + datetime.timedelta(days=1)) self.now += datetime.timedelta(days=2) last_run = next_run class GeneralSchedulerDSTTest(testingutils.MockTimeTestCase): now = datetime.datetime(2011, 11, 6, 1, 10, 0) now_utc = timeutils.current_time(tz=pytz.timezone("UTC")) def hours_until_time(self, run_time, sch): tz = sch.time_zone now = timeutils.current_time() now = tz.localize(now) if tz else now seconds = timeutils.delta_total_seconds(run_time - now) return round(max(0, seconds) / 60 / 60, 1) def hours_diff_at_datetime(self, sch, *args, **kwargs): """Return the number of hours until the next *two* runs of a job with the given scheduler """ self.now = datetime.datetime(*args, **kwargs) next_run = sch.next_run_time(self.now) t1 = self.hours_until_time(next_run, sch) next_run = sch.next_run_time(next_run.replace(tzinfo=None)) t2 = self.hours_until_time(next_run, sch) return t1, t2 def _assert_range(self, x, lower, upper): assert_gt(x, lower) assert_lt(x, upper) def test_fall_back(self): """This test checks the behavior of the scheduler at the daylight savings time 'fall back' point, when the system time zone changes from (e.g.) PDT to PST. """ sch = scheduler.GeneralScheduler(time_zone=pytz.timezone("US/Pacific")) # Exact crossover time: # datetime.datetime(2011, 11, 6, 9, 0, 0, tzinfo=pytz.utc) # This test will use times on either side of it. # From the PDT vantage point, the run time is 24.2 hours away: s1a, s1b = self.hours_diff_at_datetime(sch, 2011, 11, 6, 0, 50, 0) # From the PST vantage point, the run time is 22.8 hours away: # (this is measured from the point in absolute time 20 minutes after # the other measurement) s2a, s2b = self.hours_diff_at_datetime(sch, 2011, 11, 6, 1, 10, 0) self._assert_range(s1b - s1a, 23.99, 24.11) self._assert_range(s2b - s2a, 23.99, 24.11) self._assert_range(s1a - s2a, 1.39, 1.41) def test_correct_time(self): sch = scheduler.GeneralScheduler(time_zone=pytz.timezone("US/Pacific")) next_run_time = sch.next_run_time(self.now) assert_equal(next_run_time.hour, 0) def test_spring_forward(self): """This test checks the behavior of the scheduler at the daylight savings time 'spring forward' point, when the system time zone changes from (e.g.) PST to PDT. """ sch = scheduler.GeneralScheduler(time_zone=pytz.timezone("US/Pacific")) # Exact crossover time: # datetime.datetime(2011, 3, 13, 2, 0, 0, tzinfo=pytz.utc) # This test will use times on either side of it. # From the PST vantage point, the run time is 20.2 hours away: s1a, s1b = self.hours_diff_at_datetime(sch, 2011, 3, 13, 2, 50, 0) # From the PDT vantage point, the run time is 20.8 hours away: # (this is measured from the point in absolute time 20 minutes after # the other measurement) s2a, s2b = self.hours_diff_at_datetime(sch, 2011, 3, 13, 3, 10, 0) self._assert_range(s1b - s1a, 23.99, 24.11) self._assert_range(s2b - s2a, 23.99, 24.11) self._assert_range(s1a - s2a, -0.61, -0.59) def test_handles_tz_specific_jobs_with_tz_specific_start_time(self): sch = scheduler.GeneralScheduler(time_zone=pytz.timezone("UTC")) next_run_time = sch.next_run_time(self.now_utc) assert_equal(next_run_time.hour, 0) def test_handles_unsetting_the_time_zone(self): sch = scheduler.GeneralScheduler(time_zone=None) next_run_time = sch.next_run_time(self.now_utc) assert_equal(next_run_time.hour, 0) def test_handles_changing_the_time_zone(self): pacific_now = datetime.datetime.now(pytz.timezone("US/Pacific")) pacific_offset = pacific_now.utcoffset().total_seconds() / 60 / 60 sch = scheduler.GeneralScheduler(time_zone=pytz.timezone("US/Pacific")) next_run_time = sch.next_run_time(self.now_utc) assert_equal(next_run_time.hour, -pacific_offset) def parse_groc(config): config = schedule_parse.ConfigGenericSchedule("groc daily", config, None) return parse_groc_expression(config, NullConfigContext) def scheduler_from_config(config): return scheduler.scheduler_from_config(parse_groc(config), None) class ComplexParserTest(testingutils.MockTimeTestCase): now = datetime.datetime(2011, 6, 1) def test_parse_all(self): config_string = "1st,2nd,3rd,4th monday,Tue of march,apr,September at 00:00" cfg = parse_groc(config_string) assert_equal(cfg.ordinals, {1, 2, 3, 4}) assert_equal(cfg.monthdays, None) assert_equal(cfg.weekdays, {1, 2}) assert_equal(cfg.months, {3, 4, 9}) assert_equal(cfg.timestr, "00:00") assert_equal( scheduler_from_config(config_string), scheduler_from_config(config_string), ) def test_parse_no_weekday(self): cfg = parse_groc( "1st,2nd,3rd,10th day of march,apr,September at 00:00", ) assert_equal(cfg.ordinals, None) assert_equal(cfg.monthdays, {1, 2, 3, 10}) assert_equal(cfg.weekdays, None) assert_equal(cfg.months, {3, 4, 9}) assert_equal(cfg.timestr, "00:00") def test_parse_no_month(self): cfg = parse_groc("1st,2nd,3rd,10th day at 00:00") assert_equal(cfg.ordinals, None) assert_equal(cfg.monthdays, {1, 2, 3, 10}) assert_equal(cfg.weekdays, None) assert_equal(cfg.months, None) assert_equal(cfg.timestr, "00:00") def test_parse_monthly(self): for test_str in ("1st day", "1st day of month"): cfg = parse_groc(test_str) assert_equal(cfg.ordinals, None) assert_equal(cfg.monthdays, {1}) assert_equal(cfg.weekdays, None) assert_equal(cfg.months, None) assert_equal(cfg.timestr, "00:00") def test_wildcards(self): cfg = parse_groc("every day") assert_equal(cfg.ordinals, None) assert_equal(cfg.monthdays, None) assert_equal(cfg.weekdays, None) assert_equal(cfg.months, None) assert_equal(cfg.timestr, "00:00") def test_daily(self): sch = scheduler_from_config("every day") next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal(next_run_date.month, 6) assert_equal(next_run_date.day, 2) assert_equal(next_run_date.hour, 0) def test_daily_with_time(self): sch = scheduler_from_config("every day at 02:00") next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal(next_run_date.year, self.now.year) assert_equal(next_run_date.month, 6) assert_equal(next_run_date.day, 1) assert_equal(next_run_date.hour, 2) assert_equal(next_run_date.minute, 0) def test_weekly(self): sch = scheduler_from_config("every monday at 01:00") next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal( calendar.weekday( next_run_date.year, next_run_date.month, next_run_date.day, ), 0, ) def test_weekly_in_month(self): sch = scheduler_from_config("every monday of January at 00:01") next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal(next_run_date.year, self.now.year + 1) assert_equal(next_run_date.month, 1) assert_equal(next_run_date.hour, 0) assert_equal(next_run_date.minute, 1) assert_equal( calendar.weekday( next_run_date.year, next_run_date.month, next_run_date.day, ), 0, ) def test_monthly(self): sch = scheduler_from_config("1st day") next_run_date = sch.next_run_time(None) assert_gt(next_run_date, self.now) assert_equal(next_run_date.month, 7) if __name__ == "__main__": run() ================================================ FILE: tests/serialize/__init__.py ================================================ ================================================ FILE: tests/serialize/filehandler_test.py ================================================ import os import shutil import time from tempfile import mkdtemp from tempfile import NamedTemporaryFile from unittest import mock from testifycompat import assert_equal from testifycompat import assert_in from testifycompat import assert_not_equal from testifycompat import assert_not_in from testifycompat import run from testifycompat import setup from testifycompat import suite from testifycompat import teardown from testifycompat import TestCase from tron.serialize.filehandler import FileHandleManager from tron.serialize.filehandler import NullFileHandle from tron.serialize.filehandler import OutputPath from tron.serialize.filehandler import OutputStreamSerializer class TestFileHandleWrapper(TestCase): @setup def setup_fh_wrapper(self): self.file = NamedTemporaryFile("r") self.manager = FileHandleManager.get_instance() self.fh_wrapper = self.manager.open(self.file.name) @teardown def teardown_fh_wrapper(self): self.fh_wrapper.close() FileHandleManager.reset() def test_init(self): assert_equal(self.fh_wrapper._fh, NullFileHandle) def test_close(self): # Test close without a write, no exception is good self.fh_wrapper.close() # Test close again, after already closed self.fh_wrapper.close() def test_close_with_write(self): # Test close with a write self.fh_wrapper.write("some things") self.fh_wrapper.close() assert_equal(self.fh_wrapper._fh, NullFileHandle) assert_equal(self.fh_wrapper.manager, self.manager) # This is somewhat coupled assert_not_in(self.fh_wrapper, self.manager.cache) def test_write(self): # Test write without a previous open before_time = time.time() self.fh_wrapper.write("some things") after_time = time.time() assert self.fh_wrapper._fh assert_equal(self.fh_wrapper._fh.closed, False) assert before_time <= self.fh_wrapper.last_accessed <= after_time # Test write after previous open before_time = time.time() self.fh_wrapper.write("\nmore things") after_time = time.time() assert before_time <= self.fh_wrapper.last_accessed <= after_time self.fh_wrapper.close() with open(self.file.name) as fh: assert_equal(fh.read(), "some things\nmore things") def test_close_many(self): self.fh_wrapper.write("some things") self.fh_wrapper.close() self.fh_wrapper.close() def test_context_manager(self): with self.fh_wrapper as fh: fh.write("123") assert fh._fh is None with open(self.file.name) as fh: assert_equal(fh.read(), "123") class TestFileHandleManager(TestCase): @setup def setup_fh_manager(self): FileHandleManager.reset() self.file1 = NamedTemporaryFile("r") self.file2 = NamedTemporaryFile("r") FileHandleManager.set_max_idle_time(2) self.manager = FileHandleManager.get_instance() @teardown def teardown_fh_manager(self): FileHandleManager.reset() def test_get_instance(self): assert_equal(self.manager, FileHandleManager.get_instance()) # Repeat for good measure assert_equal(self.manager, FileHandleManager.get_instance()) def test_set_max_idle_time(self): max_idle_time = 300 FileHandleManager.set_max_idle_time(max_idle_time) assert_equal(max_idle_time, self.manager.max_idle_time) def test_open(self): # Not yet in cache fh_wrapper = self.manager.open(self.file1.name) assert_in(fh_wrapper.name, self.manager.cache) # Should now be in cache fh_wrapper2 = self.manager.open(self.file1.name) # Same wrapper assert_equal(fh_wrapper, fh_wrapper2) # Different wrapper assert_not_equal(fh_wrapper, self.manager.open(self.file2.name)) def test_cleanup_none(self): # Nothing to remove fh_wrapper = self.manager.open(self.file1.name) self.manager.cleanup() assert_in(fh_wrapper.name, self.manager.cache) def test_cleanup_single(self): fh_wrapper = self.manager.open(self.file1.name) fh_wrapper.last_accessed = 123456 def time_func(): return 123458.1 self.manager.cleanup(time_func) assert_not_in(fh_wrapper.name, self.manager.cache) assert_equal(len(self.manager.cache), 0) def test_cleanup_many(self): fh_wrappers = [ self.manager.open(self.file1.name), self.manager.open(self.file2.name), self.manager.open(NamedTemporaryFile("r").name), self.manager.open(NamedTemporaryFile("r").name), self.manager.open(NamedTemporaryFile("r").name), ] for i, fh_wrapper in enumerate(fh_wrappers): fh_wrapper.last_accessed = 123456 + i def time_func(): return 123460.1 self.manager.cleanup(time_func) assert_equal(len(self.manager.cache), 2) for fh_wrapper in fh_wrappers[:3]: assert_not_in(fh_wrapper.name, self.manager.cache) for fh_wrapper in fh_wrappers[3:]: assert_in(fh_wrapper.name, self.manager.cache) def test_cleanup_opened(self): fh_wrapper = self.manager.open(self.file1.name) fh_wrapper.write("Some things") fh_wrapper.last_accessed = 123456 def time_func(): return 123458.1 self.manager.cleanup(time_func) assert_not_in(fh_wrapper.name, self.manager.cache) assert_equal(len(self.manager.cache), 0) def test_cleanup_natural(self): FileHandleManager.set_max_idle_time(1) fh_wrapper1 = self.manager.open(self.file1.name) fh_wrapper2 = self.manager.open(self.file2.name) fh_wrapper1.write("Some things") time.sleep(1.5) fh_wrapper2.write("Other things.") assert_not_in(fh_wrapper1.name, self.manager.cache) assert_in(fh_wrapper2.name, self.manager.cache) # Now that 1 is closed, try writing again fh_wrapper1.write("Some things") assert_in(fh_wrapper1.name, self.manager.cache) assert not fh_wrapper1._fh.closed def test_remove(self): # In cache fh_wrapper = self.manager.open(self.file1.name) assert_in(fh_wrapper.name, self.manager.cache) self.manager.remove(fh_wrapper) assert_not_in(fh_wrapper.name, self.manager.cache) # Not in cache self.manager.remove(fh_wrapper) assert_not_in(fh_wrapper.name, self.manager.cache) def test_update(self): fh_wrapper1 = self.manager.open(self.file1.name) fh_wrapper2 = self.manager.open(self.file2.name) assert_equal( list(self.manager.cache.keys()), [fh_wrapper1.name, fh_wrapper2.name], ) self.manager.update(fh_wrapper1) assert_equal( list(self.manager.cache.keys()), [fh_wrapper2.name, fh_wrapper1.name], ) class TestOutputStreamSerializer(TestCase): @setup def setup_serializer(self): self.test_dir = mkdtemp() self.serial = OutputStreamSerializer([self.test_dir]) self.filename = "STARS" self.content = "123\n456\n789" self.expected = [line for line in self.content.split("\n")] @teardown def teardown_test_dir(self): shutil.rmtree(self.test_dir) def _write_contents(self): with open(self.serial.full_path(self.filename), "w") as f: f.write(self.content) def test_open(self): with self.serial.open(self.filename) as fh: fh.write(self.content) with open(self.serial.full_path(self.filename)) as f: assert_equal(f.read(), self.content) @suite("integration") def test_init_with_output_path(self): path = OutputPath(self.test_dir, "one", "two", "three") stream = OutputStreamSerializer(path) assert_equal(stream.base_path, str(path)) def test_tail(self): self._write_contents() assert_equal(self.serial.tail(self.filename), self.expected) def test_tail_num_lines(self): self._write_contents() assert_equal(self.serial.tail(self.filename, 1), self.expected[-1:]) def test_tail_file_does_not_exist(self): file_dne = "bogusfile123" assert_equal(self.serial.tail(file_dne), []) class TestOutputPath(TestCase): @setup def setup_path(self): self.path = OutputPath("one", "two", "three") def test__init__(self): assert_equal(self.path.base, "one") assert_equal(self.path.parts, ["two", "three"]) path = OutputPath("base") assert_equal(path.base, "base") assert_equal(path.parts, []) def test__iter__(self): assert_equal(list(self.path), ["one", "two", "three"]) def test__str__(self): # Breaks in windows probably, assert_equal("one/two/three", str(self.path)) def test_append(self): self.path.append("four") assert_equal(self.path.parts, ["two", "three", "four"]) def test_clone(self): new_path = self.path.clone() assert_equal(str(new_path), str(self.path)) self.path.append("alpha") assert_equal(str(new_path), "one/two/three") new_path.append("beta") assert_equal(str(self.path), "one/two/three/alpha") def test_clone_with_parts(self): new_path = self.path.clone("seven", "ten") assert_equal(list(new_path), ["one/two/three", "seven", "ten"]) def test_delete(self): tmp_dir = mkdtemp() path = OutputPath(tmp_dir) path.delete() assert not os.path.exists(tmp_dir) def test__eq__(self): other = mock.MagicMock(base="one", parts=["two", "three"]) assert_equal(self.path, other) def test__ne__(self): other = mock.MagicMock(base="one/two", parts=["three"]) assert_not_equal(self.path, other) if __name__ == "__main__": run() ================================================ FILE: tests/serialize/runstate/__init__.py ================================================ ================================================ FILE: tests/serialize/runstate/dynamodb_state_store_test.py ================================================ import gzip import json from unittest import mock import boto3 import pytest from boto3.dynamodb.types import Binary from moto import mock_dynamodb from moto.dynamodb.responses import dynamo_json_dump from tron.serialize.runstate.dynamodb_state_store import DynamoDBStateStore from tron.serialize.runstate.dynamodb_state_store import MAX_UNPROCESSED_KEYS_RETRIES def mock_transact_write_items(self): """ This mocks moto.dynamodb2.responses.DynamoHandler.transact_write_items, which is used to mock dynamodb client. This function calls put_item, update_item, and delete_item based on the arguments of transact_write_item. """ def put_item(item): name = item["TableName"] record = item["Item"] return self.dynamodb_backend.put_item(name, record) def delete_item(item): name = item["TableName"] keys = item["Key"] return self.dynamodb_backend.delete_item(name, keys) def update_item(item): name = item["TableName"] key = item["Key"] update_expression = item.get("UpdateExpression") attribute_updates = item.get("AttributeUpdates") expression_attribute_names = item.get("ExpressionAttributeNames", {}) expression_attribute_values = item.get("ExpressionAttributeValues", {}) return self.dynamodb_backend.update_item( name, key, update_expression, attribute_updates, expression_attribute_names, expression_attribute_values, ) transact_items = self.body["TransactItems"] for transact_item in transact_items: if "Put" in transact_item: put_item(transact_item["Put"]) elif "Update" in transact_item: update_item(transact_item["Update"]) elif "Delete" in transact_item: delete_item(transact_item["Delete"]) return dynamo_json_dump({}) @pytest.fixture(autouse=True) def store(): with mock.patch( "moto.dynamodb.responses.DynamoHandler.transact_write_items", new=mock_transact_write_items, create=True, ), mock_dynamodb(): dynamodb = boto3.resource("dynamodb", region_name="us-west-2") table_name = "tmp" store = DynamoDBStateStore(table_name, "us-west-2", stopping=True) store.table = dynamodb.create_table( TableName=table_name, KeySchema=[ { "AttributeName": "key", "KeyType": "HASH", }, # Partition key { "AttributeName": "index", "KeyType": "RANGE", }, # Sort key ], AttributeDefinitions=[ { "AttributeName": "key", "AttributeType": "S", }, { "AttributeName": "index", "AttributeType": "N", }, ], ProvisionedThroughput={ "ReadCapacityUnits": 10, "WriteCapacityUnits": 10, }, ) store.client = boto3.client("dynamodb", region_name="us-west-2") # Has to be yield here for moto to work yield store @pytest.fixture def small_job(): yield { "enabled": True, "run_nums": [1], } @pytest.fixture def small_object(): yield { "job_name": "example_job", "run_num": 1, "run_time": None, "time_zone": None, "node_name": "example_node", "runs": [], "cleanup_run": None, "manual": False, } @pytest.fixture def large_object(): # We need this object to exceed OBJECT_SIZE after gzip compression so it # requires multiple partitions. We pad node_name because JobRun.to_json() # only serializes known fields — arbitrary fields like "large_data" would # be silently dropped. Ideally we'd use a more realistic field like # "runs", but each action run requires ~15 nested fields, making the # fixture unwieldy. node_name is the simplest string field that survives # the JSON round-trip. yield { "job_name": "example_job", "run_num": 1, "run_time": None, "time_zone": None, "node_name": "".join(str(i) for i in range(200_000)), "runs": [], "cleanup_run": None, "manual": False, } @pytest.mark.usefixtures("store") class TestDynamoDBStateStore: def test_save(self, store, small_job, small_object): key_value_pairs = [ ( store.build_key("job_state", "two"), small_job, ), ( store.build_key("job_run_state", "four"), small_object, ), ] store.save(key_value_pairs) store._consume_save_queue() assert store.save_errors == 0 keys = [ store.build_key("job_state", "two"), store.build_key("job_run_state", "four"), ] with mock.patch("tron.config.static_config.load_yaml_file", autospec=True), mock.patch( "tron.config.static_config.build_configuration_watcher", autospec=True ): vals = store.restore(keys) for key, value in key_value_pairs: assert vals[key] == value expected_values = {keys[0]: small_job, keys[1]: small_object} for key in keys: item = store.table.get_item(Key={"key": key, "index": 0}) assert "Item" in item assert "json_val" in item["Item"] compressed_val = item["Item"]["json_val"] assert isinstance(compressed_val, Binary) decompressed_json = gzip.decompress(compressed_val.value) assert json.loads(decompressed_json) == expected_values[key] def test_save_multi_partition_object(self, store, large_object): key_value_pairs = [ ( store.build_key("job_run_state", "two"), large_object, ), ] store.save(key_value_pairs) store._consume_save_queue() assert store.save_errors == 0 keys = [store.build_key("job_run_state", "two")] with mock.patch("tron.config.static_config.load_yaml_file", autospec=True), mock.patch( "tron.config.static_config.build_configuration_watcher", autospec=True ): vals = store.restore(keys) for key, value in key_value_pairs: assert vals[key] == value def test_restore(self, store, small_object): keys = [store.build_key("job_run_state", i) for i in range(3)] value = small_object pairs = zip(keys, (value for i in range(len(keys)))) store.save(pairs) store._consume_save_queue() assert store.save_errors == 0 with mock.patch("tron.config.static_config.load_yaml_file", autospec=True), mock.patch( "tron.config.static_config.build_configuration_watcher", autospec=True ): vals = store.restore(keys) for key in keys: assert vals[key] == small_object def test_restore_multi_partition_object(self, store, large_object): keys = [store.build_key("job_run_state", i) for i in range(3)] value = large_object pairs = zip(keys, (value for i in range(len(keys)))) store.save(pairs) store._consume_save_queue() assert store.save_errors == 0 for key in keys: num_partitions, num_json_val_partitions = store._get_num_of_partitions(key) assert num_json_val_partitions > 1 assert num_partitions > 1 with mock.patch("tron.config.static_config.load_yaml_file", autospec=True), mock.patch( "tron.config.static_config.build_configuration_watcher", autospec=True ): vals = store.restore(keys) for key in keys: assert vals[key] == large_object def test_delete_item(self, store, small_object): keys = [store.build_key("job_state", i) for i in range(3)] pairs = list(zip(keys, (small_object for _ in range(len(keys))))) store.save(pairs) store._consume_save_queue() for key, _ in pairs: store._delete_item(key) for key, _ in pairs: num_partitions, num_json_val_partitions = store._get_num_of_partitions(key) assert num_partitions == 0 assert num_json_val_partitions == 0 def test_delete_multi_partition_item(self, store, large_object): keys = [store.build_key("job_state", i) for i in range(3)] pairs = list(zip(keys, (large_object for _ in range(len(keys))))) store.save(pairs) store._consume_save_queue() for key, _ in pairs: num_partitions, num_json_val_partitions = store._get_num_of_partitions(key) assert num_partitions > 1 assert num_json_val_partitions > 1 for key, _ in pairs: store._delete_item(key) for key, _ in pairs: num_partitions, num_json_val_partitions = store._get_num_of_partitions(key) assert num_partitions == 0 assert num_json_val_partitions == 0 def test_delete_if_val_is_none(self, store, small_object): key_value_pairs = [ ( store.build_key("job_state", "two"), small_object, ), ( store.build_key("job_run_state", "four"), small_object, ), ] store.save(key_value_pairs) store._consume_save_queue() delete = [ ( store.build_key("job_state", "two"), None, ), ] store.save(delete) store._consume_save_queue() assert store.save_errors == 0 # Try to restore both, we should just get one back keys = [ store.build_key("job_state", "two"), store.build_key("job_run_state", "four"), ] with mock.patch("tron.config.static_config.load_yaml_file", autospec=True), mock.patch( "tron.config.static_config.build_configuration_watcher", autospec=True ): vals = store.restore(keys) assert vals == {"job_run_state four": small_object} @pytest.mark.parametrize( "test_object, side_effects, expected_save_errors, expected_queue_length", [ # All attempts fail ("small_object", [KeyError("foo")] * 3, 3, 1), ("large_object", [KeyError("foo")] * 3, 3, 1), # Failure followed by success ("small_object", [KeyError("foo"), {}], 0, 0), ("large_object", [KeyError("foo")] + [{}] * 10, 0, 0), ], ) def test_retry_saving( self, test_object, side_effects, expected_save_errors, expected_queue_length, store, small_object, large_object ): object_mapping = { "small_object": small_object, "large_object": large_object, } value = object_mapping[test_object] with mock.patch.object( store.client, "transact_write_items", side_effect=side_effects, ) as mock_transact_write: keys = [store.build_key("job_state", 0)] pairs = zip(keys, [value]) store.save(pairs) for _ in side_effects: store._consume_save_queue() assert mock_transact_write.called assert store.save_errors == expected_save_errors assert len(store.save_queue) == expected_queue_length @pytest.mark.parametrize( "attempt, expected_delay", [ (1, 1), (2, 2), (3, 4), (4, 8), (5, 10), (6, 10), (7, 10), ], ) def test_calculate_backoff_delay(self, store, attempt, expected_delay): delay = store._calculate_backoff_delay(attempt) assert delay == expected_delay def test_retry_reading(self, store): unprocessed_value = { "Responses": {}, "UnprocessedKeys": { store.name: { "Keys": [{"key": {"S": store.build_key("job_state", 0)}, "index": {"N": "0"}}], "ConsistentRead": True, } }, } keys = [store.build_key("job_state", 0)] with mock.patch.object( store.client, "batch_get_item", return_value=unprocessed_value, ) as mock_batch_get_item, mock.patch("time.sleep") as mock_sleep, pytest.raises(Exception) as exec_info: store.restore(keys) assert "failed to retrieve items with keys" in str(exec_info.value) assert mock_batch_get_item.call_count == MAX_UNPROCESSED_KEYS_RETRIES assert mock_sleep.call_count == MAX_UNPROCESSED_KEYS_RETRIES def test_restore_exception_propagation(self, store): # This test is to ensure that restore propagates exceptions upwards: see DAR-2328 keys = [store.build_key("job_state", i) for i in range(3)] mock_future = mock.MagicMock() mock_future.result.side_effect = Exception("mocked exception") with mock.patch("concurrent.futures.Future", return_value=mock_future, autospec=True): with mock.patch("concurrent.futures.as_completed", return_value=[mock_future], autospec=True): with pytest.raises(Exception) as exec_info, mock.patch( "tron.config.static_config.load_yaml_file", autospec=True ), mock.patch("tron.config.static_config.build_configuration_watcher", autospec=True): store.restore(keys) assert str(exec_info.value) == "mocked exception" def test_serialization_failure_preserves_existing_row(self, store, small_object): """When json serialization fails, the existing DynamoDB row should not be deleted.""" key = store.build_key("job_run_state", "preserve_me") store.save([(key, small_object)]) store._consume_save_queue() assert store.save_errors == 0 new_val = {**small_object, "run_num": 999} with mock.patch.object(store, "_serialize_item", return_value=None): store.save([(key, new_val)]) # The save() call above already called _serialize_item (which returned None), # so the queue has (new_val, None). _consume_save_queue should retry and still fail. with mock.patch.object(store, "_serialize_item", return_value=None): store._consume_save_queue() # The item should be requeued assert len(store.save_queue) == 1 assert store.save_errors == 1 # The original key should still be intact with mock.patch("tron.config.static_config.load_yaml_file", autospec=True), mock.patch( "tron.config.static_config.build_configuration_watcher", autospec=True ): vals = store.restore([key]) assert vals[key] == small_object def test_serialization_retry_succeeds(self, store, small_object): """When initial json_val is None but retry succeeds, the item should be saved normally.""" key = store.build_key("job_run_state", "retry_ok") with mock.patch.object(store, "_serialize_item", return_value=None): store.save([(key, small_object)]) queued_val, queued_json = store.save_queue[key] assert queued_val == small_object assert queued_json is None # This time it should succeed store._consume_save_queue() assert len(store.save_queue) == 0 assert store.save_errors == 0 with mock.patch("tron.config.static_config.load_yaml_file", autospec=True), mock.patch( "tron.config.static_config.build_configuration_watcher", autospec=True ): vals = store.restore([key]) assert vals[key] == small_object def test_delete_sentinel_proceeds_with_deletion(self, store, small_object): """When val is None the row should be deleted.""" key = store.build_key("job_run_state", "delete_me") store.save([(key, small_object)]) store._consume_save_queue() assert store.save_errors == 0 store.save([(key, None)]) store._consume_save_queue() assert store.save_errors == 0 num_partitions, num_json_val_partitions = store._get_num_of_partitions(key) assert num_partitions == 0 assert num_json_val_partitions == 0 ================================================ FILE: tests/serialize/runstate/shelvestore_test.py ================================================ import os import shutil import tempfile from testifycompat import assert_equal from testifycompat import run from testifycompat import setup from testifycompat import teardown from testifycompat import TestCase from tron.serialize.runstate.shelvestore import Py2Shelf from tron.serialize.runstate.shelvestore import ShelveKey from tron.serialize.runstate.shelvestore import ShelveStateStore class TestShelveStateStore(TestCase): @setup def setup_store(self): self.tmpdir = tempfile.mkdtemp() self.filename = os.path.join(self.tmpdir, "state") self.store = ShelveStateStore(self.filename) @teardown def teardown_store(self): shutil.rmtree(self.tmpdir) def test__init__(self): assert_equal(self.filename, self.store.filename) def test_save(self): key_value_pairs = [ ( ShelveKey("one", "two"), { "this": "data", }, ), ( ShelveKey("three", "four"), { "this": "data2", }, ), ] self.store.save(key_value_pairs) self.store.cleanup() stored_data = Py2Shelf(self.filename) for key, value in key_value_pairs: assert_equal(stored_data[str(key.key)], value) stored_data.close() def test_delete(self): key_value_pairs = [ ( ShelveKey("one", "two"), { "this": "data", }, ), ( ShelveKey("three", "four"), { "this": "data2", }, ), # Delete first key ( ShelveKey("one", "two"), None, ), ] self.store.save(key_value_pairs) self.store.cleanup() stored_data = Py2Shelf(self.filename) assert stored_data == { str(ShelveKey("three", "four").key): {"this": "data2"}, } stored_data.close() def test_restore(self): self.store.cleanup() keys = [ShelveKey("thing", i) for i in range(5)] value = {"this": "data"} store = Py2Shelf(self.filename) for key in keys: store[str(key.key)] = value store.close() self.store.shelve = Py2Shelf(self.filename) retrieved_data = self.store.restore(keys) for key in keys: assert_equal(retrieved_data[key], value) if __name__ == "__main__": run() ================================================ FILE: tests/serialize/runstate/statemanager_test.py ================================================ import os import shutil import tempfile from unittest import mock from testifycompat import assert_equal from testifycompat import run from testifycompat import setup from testifycompat import TestCase from tests.assertions import assert_raises from tests.testingutils import autospec_method from tron.config import schema from tron.core.job import Job from tron.core.jobrun import JobRun from tron.mesos import MesosClusterRepository from tron.serialize import runstate from tron.serialize.runstate.shelvestore import ShelveStateStore from tron.serialize.runstate.statemanager import PersistenceManagerFactory from tron.serialize.runstate.statemanager import PersistenceStoreError from tron.serialize.runstate.statemanager import PersistentStateManager from tron.serialize.runstate.statemanager import StateChangeWatcher from tron.serialize.runstate.statemanager import StateSaveBuffer class TestPersistenceManagerFactory(TestCase): def test_from_config_shelve(self): tmpdir = tempfile.mkdtemp() try: fname = os.path.join(tmpdir, "state") config = schema.ConfigState( store_type="shelve", name=fname, buffer_size=0, ) manager = PersistenceManagerFactory.from_config(config) store = manager._impl assert_equal(store.filename, config.name) assert isinstance(store, ShelveStateStore) finally: shutil.rmtree(tmpdir) class TestStateSaveBuffer(TestCase): @setup def setup_buffer(self): self.buffer_size = 5 self.buffer = StateSaveBuffer(self.buffer_size) def test_save(self): assert self.buffer.save(1, 2) assert not self.buffer.save(1, 3) assert not self.buffer.save(1, 4) assert not self.buffer.save(1, 5) assert not self.buffer.save(1, 6) assert self.buffer.save(1, 7) assert_equal(self.buffer.buffer[1], 7) def test__iter__(self): self.buffer.save(1, 2) self.buffer.save(2, 3) items = list(self.buffer) assert not self.buffer.buffer assert_equal(items, [(1, 2), (2, 3)]) class TestPersistentStateManager(TestCase): @setup def setup_manager(self): self.store = mock.Mock() self.store.build_key.side_effect = lambda t, i: f"{t}{i}" self.buffer = StateSaveBuffer(1) self.manager = PersistentStateManager(self.store, self.buffer) def test__init__(self): assert_equal(self.manager._impl, self.store) def test_keys_for_items(self): names = ["namea", "nameb"] key_to_item_map = self.manager._keys_for_items("type", names) keys = ["type%s" % name for name in names] assert_equal(key_to_item_map, dict(zip(keys, names))) def test_restore(self): job_names = ["one", "two"] with mock.patch.object(self.manager, "_restore_dicts", autospec=True,) as mock_restore_dicts, mock.patch.object( self.manager, "_restore_runs_for_job", autospec=True, ) as mock_restore_runs: mock_restore_dicts.side_effect = [ # _restore_dicts for JOB_STATE { "one": {"key": "val1"}, "two": {"key": "val2"}, }, ] restored_state = self.manager.restore(job_names) assert mock_restore_dicts.call_args_list == [ mock.call(runstate.JOB_STATE, job_names), ] assert len(mock_restore_runs.call_args_list) == 2 assert restored_state == { runstate.JOB_STATE: { "one": {"key": "val1", "runs": mock_restore_runs.return_value}, "two": {"key": "val2", "runs": mock_restore_runs.return_value}, }, } def test_restore_runs_for_job(self): job_state = {"run_nums": [2, 3], "enabled": True} with mock.patch.object( self.manager, "_restore_dicts", autospec=True, ) as mock_restore_dicts: mock_restore_dicts.side_effect = [ {"job_a.2": {"job_name": "job_a", "run_num": 2}, "job_a.3": {"job_name": "job_a", "run_num": 3}} ] runs = self.manager._restore_runs_for_job("job_a", job_state) assert mock_restore_dicts.call_args_list == [mock.call(runstate.JOB_RUN_STATE, ["job_a.2", "job_a.3"])] assert runs == [{"job_name": "job_a", "run_num": 3}, {"job_name": "job_a", "run_num": 2}] def test_restore_runs_for_job_one_missing(self): job_state = {"run_nums": [2, 3], "enabled": True} with mock.patch.object( self.manager, "_restore_dicts", autospec=True, ) as mock_restore_dicts: mock_restore_dicts.side_effect = [{"job_a.3": {"job_name": "job_a", "run_num": 3}, "job_b": {}}] runs = self.manager._restore_runs_for_job("job_a", job_state) assert mock_restore_dicts.call_args_list == [ mock.call(runstate.JOB_RUN_STATE, ["job_a.2", "job_a.3"]), ] assert runs == [{"job_name": "job_a", "run_num": 3}] def test_restore_dicts(self): names = ["namea", "nameb"] autospec_method(self.manager._keys_for_items) self.manager._keys_for_items.return_value = dict(enumerate(names)) self.store.restore.return_value = { 0: { "state": "data", }, 1: { "state": "2data", }, } state_data = self.manager._restore_dicts("type", names) expected = { names[0]: { "state": "data", }, names[1]: { "state": "2data", }, } assert_equal(expected, state_data) def test_save(self): name, state_data = "name", mock.Mock() self.manager.save(runstate.JOB_STATE, name, state_data) key = f"{runstate.JOB_STATE}{name}" self.store.save.assert_called_with([(key, state_data)]) def test_save_failed(self): self.store.save.side_effect = PersistenceStoreError("blah") assert_raises( PersistenceStoreError, self.manager.save, None, None, None, ) def test_save_while_disabled(self): with self.manager.disabled(): self.manager.save("something", "name", mock.Mock()) assert not self.store.save.mock_calls def test_delete(self): name = "name" self.manager.delete(runstate.JOB_STATE, name) key = f"{runstate.JOB_STATE}{name}" self.store.save.assert_called_with([(key, None)]) def test_cleanup(self): self.manager.cleanup() self.store.cleanup.assert_called_with() def test_disabled(self): with self.manager.disabled(): assert not self.manager.enabled assert self.manager.enabled def test_disabled_with_exception(self): def testfunc(): with self.manager.disabled(): raise ValueError() assert_raises(ValueError, testfunc) assert self.manager.enabled def test_disabled_nested(self): self.manager.enabled = False with self.manager.disabled(): pass assert not self.manager.enabled class TestStateChangeWatcher(TestCase): @setup def setup_watcher(self): self.watcher = StateChangeWatcher() self.state_manager = mock.create_autospec(PersistentStateManager) self.watcher.state_manager = self.state_manager def test_update_from_config_no_change(self): self.watcher.config = state_config = mock.Mock() assert not self.watcher.update_from_config(state_config) autospec_method(self.watcher.shutdown) assert_equal(self.watcher.state_manager, self.state_manager) assert not self.watcher.shutdown.mock_calls @mock.patch( "tron.serialize.runstate.statemanager.PersistenceManagerFactory", autospec=True, ) def test_update_from_config_changed(self, mock_factory): state_config = mock.Mock() autospec_method(self.watcher.shutdown) assert self.watcher.update_from_config(state_config) assert_equal(self.watcher.config, state_config) self.watcher.shutdown.assert_called_with() assert_equal( self.watcher.state_manager, mock_factory.from_config.return_value, ) mock_factory.from_config.assert_called_with(state_config) def test_save_job(self): mock_job = mock.Mock() self.watcher.save_job(mock_job) self.watcher.state_manager.save.assert_called_with( runstate.JOB_STATE, mock_job.name, mock_job.state_data, ) def test_shutdown(self): self.watcher.shutdown() assert not self.watcher.state_manager.enabled self.watcher.state_manager.cleanup.assert_called_with() def test_disabled(self): context = self.watcher.disabled() assert_equal(self.watcher.state_manager.disabled.return_value, context) def test_restore(self): jobs = mock.Mock() self.watcher.restore(jobs) self.watcher.state_manager.restore.assert_called_with(jobs) def test_handler_mesos_change(self): self.watcher.handler( observable=MesosClusterRepository, event=None, ) self.watcher.state_manager.save.assert_called_with( runstate.MESOS_STATE, MesosClusterRepository.name, MesosClusterRepository.state_data, ) def test_handler_job_state_change(self): mock_job = mock.Mock(spec_set=Job) with mock.patch.object(self.watcher, "save_job") as mock_save_job: self.watcher.handler( observable=mock_job, event=Job.NOTIFY_STATE_CHANGE, ) mock_save_job.assert_called_with(mock_job) def test_handler_job_new_run(self): mock_job = mock.Mock(spec_set=Job) mock_job_run = mock.Mock(spec_set=JobRun) with mock.patch.object(self.watcher, "save_job",) as mock_save_job, mock.patch.object( self.watcher, "watch", ) as mock_watch: # Error: No job run in event data, do nothing self.watcher.handler( observable=mock_job, event=Job.NOTIFY_NEW_RUN, ) assert mock_watch.call_count == 0 assert mock_save_job.call_count == 0 # Correct case self.watcher.handler( observable=mock_job, event=Job.NOTIFY_NEW_RUN, event_data=mock_job_run, ) mock_watch.assert_called_with(mock_job_run) assert mock_save_job.call_count == 0 def test_handler_job_run_state_change(self): mock_job_run = mock.MagicMock(spec_set=JobRun) self.watcher.handler( observable=mock_job_run, event=JobRun.NOTIFY_STATE_CHANGED, ) self.watcher.state_manager.save.assert_called_with( runstate.JOB_RUN_STATE, mock_job_run.name, mock_job_run.state_data, ) def test_handler_job_run_removed(self): mock_job_run = mock.MagicMock(spec_set=JobRun) self.watcher.handler( observable=mock_job_run, event=JobRun.NOTIFY_REMOVED, ) self.watcher.state_manager.delete.assert_called_with( runstate.JOB_RUN_STATE, mock_job_run.name, ) if __name__ == "__main__": run() ================================================ FILE: tests/serialize/runstate/yamlstore_test.py ================================================ import os import tempfile from testifycompat import assert_equal from testifycompat import run from testifycompat import setup from testifycompat import teardown from testifycompat import TestCase from tron import yaml from tron.serialize.runstate import yamlstore class TestYamlStateStore(TestCase): @setup def setup_store(self): self.filename = os.path.join(tempfile.gettempdir(), "yaml_state") self.store = yamlstore.YamlStateStore(self.filename) self.test_data = { "one": { "a": 1, }, "two": { "b": 2, }, "three": { "c": 3, }, } @teardown def teardown_store(self): try: os.unlink(self.filename) except OSError: pass def test_restore(self): with open(self.filename, "w") as fh: yaml.dump(self.test_data, fh) keys = [yamlstore.YamlKey("one", "a"), yamlstore.YamlKey("three", "c")] state_data = self.store.restore(keys) assert_equal(self.store.buffer, self.test_data) expected = {keys[0]: 1, keys[1]: 3} assert_equal(expected, state_data) def test_restore_missing_type_key(self): with open(self.filename, "w") as fh: yaml.dump(self.test_data, fh) keys = [yamlstore.YamlKey("seven", "a")] state_data = self.store.restore(keys) assert_equal(self.store.buffer, self.test_data) assert_equal({}, state_data) def test_restore_file_missing(self): state_data = self.store.restore(["some", "keys"]) assert_equal(state_data, {}) def test_save(self): expected = {"one": {"five": "dataz"}, "two": {"seven": "stars"}} key_value_pairs = [ (yamlstore.YamlKey("one", "five"), "barz"), ] # Save first self.store.save(key_value_pairs) # Save second key_value_pairs = [ (yamlstore.YamlKey("two", "seven"), "stars"), (yamlstore.YamlKey("one", "five"), "dataz"), ] self.store.save(key_value_pairs) assert_equal(self.store.buffer, expected) with open(self.filename) as fh: actual = yaml.load(fh) assert_equal(actual, expected) def test_delete(self): expected = {"state_a": {"five": "barz"}} key_value_pairs = [ (yamlstore.YamlKey("state_a", "five"), "barz"), (yamlstore.YamlKey("state_c", "five"), "delete_all_c"), (yamlstore.YamlKey("state_a", "six"), "delete_one_a"), ] # Save first self.store.save(key_value_pairs) # Save second key_value_pairs = [ (yamlstore.YamlKey("state_c", "five"), None), (yamlstore.YamlKey("state_a", "six"), None), ] self.store.save(key_value_pairs) assert_equal(self.store.buffer, expected) with open(self.filename) as fh: actual = yaml.load(fh) assert_equal(actual, expected) if __name__ == "__main__": run() ================================================ FILE: tests/ssh_test.py ================================================ from unittest import mock from twisted.python import failure from testifycompat import assert_equal from testifycompat import assert_not_equal from testifycompat import setup from testifycompat import TestCase from tests.testingutils import autospec_method from tron import ssh class TestClientTransport(TestCase): @setup def setup_transport(self): self.username = "username" self.options = mock.Mock() self.expected_pub_key = mock.Mock() self.transport = ssh.ClientTransport( self.username, self.options, self.expected_pub_key, ) def test_verifyHostKey_missing_pub_key(self): self.transport.expected_pub_key = None result = self.transport.verifyHostKey(mock.Mock(), mock.Mock()) assert_equal(result.result, 1) @mock.patch("tron.ssh.keys", autospec=True) def test_verifyHostKey_matching_pub_key(self, mock_keys): mock_keys.Key.fromString.return_value = self.expected_pub_key public_key = mock.Mock() result = self.transport.verifyHostKey(public_key, mock.Mock()) assert_equal(result.result, 2) mock_keys.Key.fromString.assert_called_with(public_key) @mock.patch("tron.ssh.keys", autospec=True) def test_verifyHostKey_mismatch_pub_key(self, _): public_key = mock.Mock() result = self.transport.verifyHostKey(public_key, mock.Mock()) assert isinstance(result.result, failure.Failure) def test_connnectionSecure(self): self.transport.connection_defer = mock.Mock() autospec_method(self.transport.requestService) self.transport.connectionSecure() conn = self.transport.connection_defer.mock_calls[0][1][0] assert isinstance(conn, ssh.ClientConnection) auth_service = self.transport.requestService.mock_calls[0][1][0] assert isinstance(auth_service, ssh.NoPasswordAuthClient) class TestSSHAuthOptions(TestCase): def test_from_config_none(self): ssh_conf = mock.Mock(agent=False, identities=[]) ssh_options = ssh.SSHAuthOptions.from_config(ssh_conf) assert_equal(ssh_options["noagent"], True) assert_equal(ssh_options.identitys, []) def test_from_config_both(self): identities = ["one", "two"] ssh_conf = mock.Mock(agent=True, identities=identities) ssh_options = ssh.SSHAuthOptions.from_config(ssh_conf) assert_equal(ssh_options["noagent"], False) assert_equal(ssh_options.identitys, identities) def test__eq__true(self): config = mock.Mock(agent=True, identities=["one", "two"]) assert_equal( ssh.SSHAuthOptions.from_config(config), ssh.SSHAuthOptions.from_config(config), ) def test__eq__false(self): config = mock.Mock(agent=True, identities=["one", "two"]) second_config = mock.Mock(agent=True, identities=["two"]) assert_not_equal( ssh.SSHAuthOptions.from_config(config), ssh.SSHAuthOptions.from_config(second_config), ) ================================================ FILE: tests/test_id_rsa ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAz+pjFOAHRLcQc6X51SyysJurwBTben3OWCG46CB81faxTVrC gcVEM3HCHz5MU8jI0Wb+DK9AXU229yQ8OCRPt3CrzxyI031ZrzuagVNRb/hmiBN+ +SNmIFQl97/bqht5DXKykUJQmP31crLz3+G0rGXONeJjZAqFFIA1NfMMAnRZMRMo 417Xf/p3yYXCV0AbWqbMWYA8aQKWFs4EY36vOzUJyftJI1cGttvskcCd3dce4DpR 8JEnI/rkRKOR19eCLCtftV6CmC3igLoSF1JDeQgkVZRdVG/pcwfV1a/SEAlxAO0e RcZccMVoPi/Ans2CxIeukQ6ThFyRMcaD+iku6QIBIwKCAQBw3lMLVQtCj0Nx+wP3 YWhRPpBvlktCfs8Z5muxNjUjsc32yt6eN+Mx3qs1iDgQOcwZ53P4QeEctSjPTi9R rU/YnEACt7f9x7RXz+Yo8rcuJ8KhpC78Rmqj1ei5sUtcWA6DpKoUVzMROWf8b8Y3 tQpO9W/xXaOrVir8gBzixcSw3xgBiobkxWeiwCrD8LsPQ7qrP27nf252wrIa8BJx 7qFrRSa05I6Hinj3jMwXUiUPVXBG5m+t8Z4c9WITvnr91uwHuHZc9TV6HedqznmA 0n6iq2rPHPj4D0gWZFKvTFXN06KD+ldAiyWWGg3nob7gzFkGZkbAk2KLUN7PLuv3 0KRPAoGBAPg8DkyznKNVgenaKt5Jc9YdLCVd4uvvjuTgmBiezvE4iwVRkIm6Q0jq FLgN4c4fUd10Fa4QPwIFXQBK5UYTwoklvDttZ+LnwY5SEzGeaXHdi8suOPlcDvDp L5Fpbi6wGJk04MXcApIwLIQCQp+pJeIHbKoq7c2ABMuqp+QPbUwrAoGBANZrcXfo p2fIxbhrdwS7yMT0jInX6HeDvDM6e67X3TZYAjZuj9PNtlg6qOT6mFWUzOvcEFzj RDfA2d53DpGfpALdQ8OBxyI7QrdcbeW0Br/pGCicW+ovbT3Z3Bfl+GOhav1EuHRR L0GlW2xN3h+oZRexUSPAA80ep9o/XGaAjGM7AoGBAOoMvQZ9dm4da9x9PlyOZeci 0doWsWIcYihBeXZMlzs1T+BxeaZtymlRu8N6zZZ1TS/ietdRJXbvHSwpW9RbxgxI JoEso8dPipwhf8+yne8EFhdXd4wGVzrqfU6WmxYTv2vhZjblYYKFMUlD9ax66TQy 4sxUXI6OpW+SRoaSM9oZAoGBAKVo0+AogSQtKtAYYyDoojGJc7rLIQu9ZUwXLDZs AmvAPDiebvPZNOT6DULtM6+77ooQKeFBmwZwMwqznYZH840uWNil8WOMzREbariD kC2lL+TQZCmvjslQSrNZolt8hbwQcQlF8UFFDAMXf3eB55XvL/cB1wvzE8Wel7zJ kN7VAoGBAIISjKvjU3VpKNX9CEQ7V6eb7OuUKo3gUTCMaunWwpVYZ4pR5UiOtib+ lcDpTQybKQOqSgHKQ13L/7Nu0GY9ILlXhfNhRlflSNUNaGcwMykxm0tzh3TsQ3vv 8cQYV+W+24bwK39tSzl+n/nnuDdly7aTS2nDrhwWIsL45DU1QXK8 -----END RSA PRIVATE KEY----- ================================================ FILE: tests/test_id_rsa.pub ================================================ ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAz+pjFOAHRLcQc6X51SyysJurwBTben3OWCG46CB81faxTVrCgcVEM3HCHz5MU8jI0Wb+DK9AXU229yQ8OCRPt3CrzxyI031ZrzuagVNRb/hmiBN++SNmIFQl97/bqht5DXKykUJQmP31crLz3+G0rGXONeJjZAqFFIA1NfMMAnRZMRMo417Xf/p3yYXCV0AbWqbMWYA8aQKWFs4EY36vOzUJyftJI1cGttvskcCd3dce4DpR8JEnI/rkRKOR19eCLCtftV6CmC3igLoSF1JDeQgkVZRdVG/pcwfV1a/SEAlxAO0eRcZccMVoPi/Ans2CxIeukQ6ThFyRMcaD+iku6Q== rhettg@devvm1 ================================================ FILE: tests/testingutils.py ================================================ import functools import logging import time from unittest import mock from testifycompat import class_setup from testifycompat import class_teardown from testifycompat import setup from testifycompat import teardown from testifycompat import TestCase from tron.utils import timeutils log = logging.getLogger(__name__) # TODO: remove when replaced with tron.eventloop class MockReactorTestCase(TestCase): """Patch the reactor to a MockReactor.""" # Override this in subclasses module_to_mock = None @class_setup def class_setup_patched_reactor(self): msg = "%s must set a module_to_mock field" % self.__class__ assert self.module_to_mock, msg self.old_reactor = getattr(self.module_to_mock, "reactor") @class_teardown def teardown_patched_reactor(self): setattr(self.module_to_mock, "reactor", self.old_reactor) @setup def setup_mock_reactor(self): self.reactor = mock.MagicMock() setattr(self.module_to_mock, "reactor", self.reactor) # TODO: remove class MockTimeTestCase(TestCase): now = None @setup def setup_current_time(self): assert self.now, "%s must set a now field" % self.__class__ self.old_current_time = timeutils.current_time timeutils.current_time = lambda tz=None: self.now @teardown def teardown_current_time(self): timeutils.current_time = self.old_current_time # Reset 'now' back to what was set on the class because some test may # have changed it self.now = self.__class__.now def retry(max_tries=3, delay=0.1, exceptions=(KeyError, IndexError)): """A function decorator for re-trying an operation. Useful for MongoDB which is only eventually consistent. """ def wrapper(f): @functools.wraps(f) def wrap(*args, **kwargs): for _ in range(max_tries): try: return f(*args, **kwargs) except exceptions: time.sleep(delay) raise return wrap return wrapper def autospec_method(method, *args, **kwargs): """create an autospec for an instance method.""" mocked_method = mock.create_autospec(method, *args, **kwargs) setattr(method.__self__, method.__name__, mocked_method) ================================================ FILE: tests/tools/sync_tron_state_from_k8s_test.py ================================================ from unittest import mock import pytest from kubernetes.client import V1ObjectMeta from kubernetes.client import V1Pod from kubernetes.client import V1PodStatus from tools.sync_tron_state_from_k8s import get_matching_pod from tools.sync_tron_state_from_k8s import get_tron_state_from_api from tools.sync_tron_state_from_k8s import update_tron_from_pods def create_mock_pod(name: str, phase: str, labels: dict[str, str], creation_timestamp: str): metadata = V1ObjectMeta(name=name, creation_timestamp=creation_timestamp, labels=labels) status = V1PodStatus(phase=phase) return V1Pod(metadata=metadata, status=status) class TestSyncTronStateFromK8s: @pytest.fixture(autouse=True) def setup_test_data(self): self.pods = { p.metadata.name: p for p in [ create_mock_pod( "service.job.2.action", "Succeeded", { "paasta.yelp.com/service": "service", "paasta.yelp.com/instance": "job.action", "tron.yelp.com/run_num": "2", }, "2024-01-01T00:00:00", ), create_mock_pod( "service.job.3.action-nomatch", "Failed", { "paasta.yelp.com/service": "service", "paasta.yelp.com/instance": "job.action", "tron.yelp.com/run_num": "3", }, "2024-01-01T00:00:00", ), create_mock_pod( "service.job.4.action-nomatch", "Failed", { "paasta.yelp.com/service": "service", "paasta.yelp.com/instance": "job.action", "tron.yelp.com/run_num": "4", }, "2024-01-01T00:00:00", ), create_mock_pod( "service.job.4.action-nomatch-retry2", "Succeeded", { "paasta.yelp.com/service": "service", "paasta.yelp.com/instance": "job.action", "tron.yelp.com/run_num": "4", }, "2024-01-01T01:00:00", ), create_mock_pod( "service.job2.10.action", "Failed", { "paasta.yelp.com/service": "service", "paasta.yelp.com/instance": "job2.action", "tron.yelp.com/run_num": "10", }, "2024-01-01T01:00:00", ), create_mock_pod( "service.job2.10.action", "Running", { "paasta.yelp.com/service": "service", "paasta.yelp.com/instance": "job2.action", "tron.yelp.com/run_num": "10", }, "2024-01-01T01:05:00", ), create_mock_pod( # Technically this pod name would not actually exist "service.job_with_an_extremely_extremely_extremely_extremely_extremely_long_name.10.action", "Succeeded", { "paasta.yelp.com/service": "service", # If PaaSTA's setup_tron_namespace changes how we create these labels, this test will need updating "paasta.yelp.com/instance": "job_with_an_extremely_extremely_extremely_extremely_extrem-26i4", "tron.yelp.com/run_num": "10", }, "2024-01-01T01:05:00", ), ] } # 1 matching pod by labels # 2 matching pod by labels # no matching pod # test matching by hashed instance name @pytest.mark.parametrize( "job_name,run_num,expected_pod_name", [ ("service.job", "3", "service.job.3.action-nomatch"), ("service.job", "4", "service.job.4.action-nomatch-retry2"), ("service.job2", "10", None), ("service2.job", "1", None), ( "service.job_with_an_extremely_extremely_extremely_extremely_extremely_long_name", "10", "service.job_with_an_extremely_extremely_extremely_extremely_extremely_long_name.10.action", ), ], ) def test_get_matching_pod(self, job_name, run_num, expected_pod_name): test_action_run = {"action_name": "action", "job_name": f"{job_name}", "run_num": run_num} matching_pod = get_matching_pod(test_action_run, self.pods) assert matching_pod == self.pods.get(expected_pod_name) # verify we send correct num_runs # verify we are sending request for jobs + one for each job @mock.patch("tools.sync_tron_state_from_k8s.get_client_config", autospec=True) @mock.patch("tools.sync_tron_state_from_k8s.Client", autospec=True) def test_get_tron_state_from_api(self, mock_client, mock_get_client_config): mock_client.return_value = mock.Mock() mock_client.return_value.jobs.return_value = [{"url": "/uri", "name": "some job"}] mock_client.return_value.job.return_value = {"runs": []} mock_get_client_config.return_value = {"server": "https://localhost:8888"} get_tron_state_from_api(None, num_runs=10) mock_client.assert_called_with("https://localhost:8888") mock_client.return_value.jobs.assert_called_with( include_job_runs=False, include_action_runs=False, include_action_graph=False, include_node_pool=False ) mock_client.return_value.job.assert_called_with("/api/uri", include_action_runs=True, count=10) @mock.patch("tools.sync_tron_state_from_k8s.subprocess.run", autospec=True) def test_update_tron(self, mock_subprocess_run): # sorry for the blob of test data tron_state = [ { "name": "service.job", "runs": [ { "runs": [ { "id": "service.job.2.action", "action_name": "action", "run_num": "2", "job_name": "service.job", "state": "unknown", } ] }, { "runs": [ { "id": "service.job.3.action", "action_name": "action", "run_num": "3", "job_name": "service.job", "state": "running", }, { "id": "service.job.3.action2", "action_name": "action2", "run_num": "3", "job_name": "service.job", "state": "running", }, ] }, { "runs": [ { "id": "service.job.4.action", "action_name": "action", "run_num": "4", "job_name": "service.job", "state": "starting", } ] }, { "runs": [ { "id": "service.job.5.action", "action_name": "action", "run_num": "5", "job_name": "service.job", "state": "starting", } ] }, ], }, { "name": "service.job2", "runs": [ { "runs": [ { "id": "service.job2.10.action", "action_name": "action", "run_num": "10", "job_name": "service.job2", "state": "succeeded", }, ] }, ], }, ] good_subprocess_run = mock.Mock(returncode=0) bad_subprocess_run = mock.Mock(returncode=1) expected_calls = [ mock.call(["tronctl", "success", "service.job.2.action"], capture_output=True, text=True), mock.call(["tronctl", "fail", "service.job.3.action"], capture_output=True, text=True), mock.call(["tronctl", "success", "service.job.4.action"], capture_output=True, text=True), ] mock_subprocess_run.return_value = good_subprocess_run result = update_tron_from_pods(tron_state, self.pods, tronctl_wrapper="tronctl", do_work=True) assert result["updated"] == ["service.job.2.action", "service.job.3.action", "service.job.4.action"] assert result["error"] == [] mock_subprocess_run.assert_has_calls(expected_calls, any_order=True) mock_subprocess_run.return_value = bad_subprocess_run result = update_tron_from_pods(tron_state, self.pods, tronctl_wrapper="tronctl", do_work=True) assert result["error"] == ["service.job.2.action", "service.job.3.action", "service.job.4.action"] ================================================ FILE: tests/trond_test.py ================================================ import datetime import textwrap from subprocess import CalledProcessError from textwrap import dedent import pytest from testifycompat import assert_equal from testifycompat import assert_gt from tests import sandbox from tron.core import actionrun BASIC_CONFIG = """ ssh_options: agent: true nodes: - name: local hostname: 'localhost' state_persistence: name: "state_data.shelve" store_type: shelve """ SINGLE_ECHO_CONFIG = ( BASIC_CONFIG + """ jobs: - name: "echo_job" node: local schedule: "cron 0 * * * *" actions: - name: "echo_action" command: "echo 'Echo!'" """ ) DOUBLE_ECHO_CONFIG = ( SINGLE_ECHO_CONFIG + """ - name: "another_echo_action" command: "echo 'Today is %(shortdate)s, which is the same as %(year)s-%(month)s-%(day)s' && false" """ ) ALT_NAMESPACED_ECHO_CONFIG = """ jobs: - name: "echo_job" node: local schedule: "cron 0 * * * *" actions: - name: "echo_action" command: "echo 'Echo!'" """ TOUCH_CLEANUP_FMT = """ cleanup_action: command: "echo 'at last'" """ @pytest.mark.skip(reason="We don't have a setup for sandbox tests yet") class TrondEndToEndTestCase(sandbox.SandboxTestCase): def test_end_to_end_basic(self): self.start_with_config(SINGLE_ECHO_CONFIG) client = self.sandbox.client assert_equal( self.client.config("MASTER")["config"], SINGLE_ECHO_CONFIG, ) # reconfigure and confirm results second_config = DOUBLE_ECHO_CONFIG + TOUCH_CLEANUP_FMT self.sandbox.tronfig(second_config) assert_equal(client.config("MASTER")["config"], second_config) # reconfigure, by uploading a third configuration self.sandbox.tronfig(ALT_NAMESPACED_ECHO_CONFIG, name="ohce") self.sandbox.client.home() # run the job and check its output echo_job_name = "MASTER.echo_job" job_url = client.get_url(echo_job_name) action_url = client.get_url("MASTER.echo_job.1.echo_action") self.sandbox.tronctl("start", echo_job_name) def wait_on_cleanup(): return ( len(client.job(job_url)["runs"]) >= 2 and client.action_runs(action_url)["state"] == actionrun.ActionRun.SUCCEEDED ) sandbox.wait_on_sandbox(wait_on_cleanup) echo_action_run = client.action_runs(action_url) another_action_url = client.get_url( "MASTER.echo_job.1.another_echo_action", ) other_act_run = client.action_runs(another_action_url) assert_equal( echo_action_run["state"], actionrun.ActionRun.SUCCEEDED, ) assert_equal(echo_action_run["stdout"], ["Echo!"]) assert_equal( other_act_run["state"], actionrun.ActionRun.FAILED, ) now = datetime.datetime.now() stdout = now.strftime( "Today is %Y-%m-%d, which is the same as %Y-%m-%d", ) assert_equal(other_act_run["stdout"], [stdout]) job_runs_url = client.get_url("%s.1" % echo_job_name) assert_equal( client.job_runs(job_runs_url)["state"], actionrun.ActionRun.FAILED, ) def test_node_reconfig(self): job_config = dedent( """ jobs: - name: a_job node: local schedule: "cron * * * * *" actions: - name: first_action command: "echo something" """ ) second_config = ( dedent( """ ssh_options: agent: true nodes: - name: local hostname: '127.0.0.1' state_persistence: name: "state_data.shelve" store_type: shelve """ ) + job_config ) self.start_with_config(BASIC_CONFIG + job_config) job_url = self.client.get_url("MASTER.a_job.0") sandbox.wait_on_state( self.client.job_runs, job_url, actionrun.ActionRun.SUCCEEDED, ) self.sandbox.tronfig(second_config) job_url = self.client.get_url("MASTER.a_job") def wait_on_next_run(): last_run = self.client.job(job_url)["runs"][0] return last_run["node"]["hostname"] == "127.0.0.1" sandbox.wait_on_sandbox(wait_on_next_run) @pytest.mark.skip(reason="We don't have a setup for sandbox tests yet") class TronCommandsTestCase(sandbox.SandboxTestCase): def test_tronview(self): self.start_with_config(SINGLE_ECHO_CONFIG) expected = """\nServices:\nNo Services\n\n\nJobs: Name State Scheduler Last Success MASTER.echo_job enabled cron 1:00:00 None """ def remove_line_space(s): return [line.replace(" ", "") for line in s.split("\n")] actual = self.sandbox.tronview()[0] assert_equal(remove_line_space(actual), remove_line_space(expected)) def test_tronctl_with_job(self): self.start_with_config(SINGLE_ECHO_CONFIG + TOUCH_CLEANUP_FMT) job_name = "MASTER.echo_job" job_url = self.client.get_url(job_name) self.sandbox.tronctl("start", job_name) cleanup_url = self.client.get_url("MASTER.echo_job.1.cleanup") sandbox.wait_on_state( self.client.action_runs, cleanup_url, actionrun.ActionRun.SUCCEEDED, ) action_run_url = self.client.get_url("MASTER.echo_job.1.echo_action") assert_equal( self.client.action_runs(action_run_url)["state"], actionrun.ActionRun.SUCCEEDED, ) job_run_url = self.client.get_url("MASTER.echo_job.1") assert_equal( self.client.job_runs(job_run_url)["state"], actionrun.ActionRun.SUCCEEDED, ) assert_equal(self.client.job(job_url)["status"], "enabled") self.sandbox.tronctl("disable", job_name) sandbox.wait_on_state(self.client.job, job_url, "disabled", "status") def test_tronfig(self): self.start_with_config(SINGLE_ECHO_CONFIG) stdout, stderr = self.sandbox.tronfig() assert_equal(stdout.rstrip(), SINGLE_ECHO_CONFIG.rstrip()) def test_tronfig_failure(self): self.start_with_config(SINGLE_ECHO_CONFIG) bad_config = "this is not valid: yaml: is it?" def test_return_code(exc): assert_equal(exc.returncode, 1) with pytest.raises(CalledProcessError): test_return_code(self.sandbox.tronfig, bad_config) @pytest.mark.skip(reason="We don't have a setup for sandbox tests yet") class JobEndToEndTestCase(sandbox.SandboxTestCase): def test_cleanup_on_failure(self): config = ( BASIC_CONFIG + dedent( """ jobs: - name: "failjob" node: local schedule: "daily 04:20" actions: - name: "failaction" command: "failplz" """ ) + TOUCH_CLEANUP_FMT ) self.start_with_config(config) action_run_url = self.client.get_url("MASTER.failjob.0.failaction") sandbox.wait_on_state( self.client.action_runs, action_run_url, actionrun.ActionRun.FAILED, ) action_run_url = self.client.get_url("MASTER.failjob.1.cleanup") sandbox.wait_on_state( self.client.action_runs, action_run_url, actionrun.ActionRun.SUCCEEDED, ) job_runs = self.client.job( self.client.get_url("MASTER.failjob"), )["runs"] assert_gt(len(job_runs), 1) def test_skip_failed_actions(self): config = BASIC_CONFIG + dedent( """ jobs: - name: "multi_step_job" node: local schedule: "daily 04:20" actions: - name: "broken" command: "failingcommand" - name: "works" command: "echo ok" requires: [broken] """ ) self.start_with_config(config) action_run_url = self.client.get_url("MASTER.multi_step_job.0.broken") waiter = sandbox.build_waiter_func( self.client.action_runs, action_run_url, ) waiter(actionrun.ActionRun.FAILED) self.sandbox.tronctl("skip", "MASTER.multi_step_job.0.broken") waiter(actionrun.ActionRun.SKIPPED) action_run_url = self.client.get_url("MASTER.multi_step_job.0.works") sandbox.wait_on_state( self.client.action_runs, action_run_url, actionrun.ActionRun.SUCCEEDED, ) job_run_url = self.client.get_url("MASTER.multi_step_job.0") sandbox.wait_on_state( self.client.job_runs, job_run_url, actionrun.ActionRun.SUCCEEDED, ) def test_failure_on_multi_step_job_doesnt_wedge_tron(self): config = BASIC_CONFIG + dedent( """ jobs: - name: "random_failure_job" node: local queueing: true schedule: "daily 04:20" actions: - name: "fa" command: "sleep 0.1; failplz" - name: "sa" command: "echo 'you will never see this'" requires: [fa] """ ) self.start_with_config(config) job_url = self.client.get_url("MASTER.random_failure_job") def wait_on_random_failure_job(): return len(self.client.job(job_url)["runs"]) >= 4 sandbox.wait_on_sandbox(wait_on_random_failure_job) job_runs = self.client.job(job_url)["runs"] expected = [actionrun.ActionRun.FAILED for _ in range(3)] assert_equal([run["state"] for run in job_runs[-3:]], expected) def test_cancel_schedules_a_new_run(self): config = BASIC_CONFIG + dedent( """ jobs: - name: "a_job" node: local schedule: "daily 05:00:00" actions: - name: "first_action" command: "echo OK" """ ) self.start_with_config(config) job_name = "MASTER.a_job" job_url = self.client.get_url(job_name) self.sandbox.tronctl("cancel", "%s.0" % job_name) def wait_on_cancel(): return len(self.client.job(job_url)["runs"]) == 2 sandbox.wait_on_sandbox(wait_on_cancel) run_states = [run["state"] for run in self.client.job(job_url)["runs"]] expected = [ actionrun.ActionRun.SCHEDULED, actionrun.ActionRun.CANCELLED, ] assert_equal(run_states, expected) def test_job_queueing_false_with_overlap(self): """Test that a job that has queueing false properly cancels an overlapping job run. """ config = BASIC_CONFIG + dedent( """ jobs: - name: "cancel_overlap" schedule: "cron * * * * *" queueing: False node: local actions: - name: "do_something" command: "sleep 3s" - name: "do_other" command: "sleep 3s" cleanup_action: command: "echo done" """ ) self.start_with_config(config) job_url = self.client.get_url("MASTER.cancel_overlap") job_run_url = self.client.get_url("MASTER.cancel_overlap.1") def wait_on_job_schedule(): return len(self.client.job(job_url)["runs"]) == 2 sandbox.wait_on_sandbox(wait_on_job_schedule) sandbox.wait_on_state( self.client.job, job_run_url, actionrun.ActionRun.CANCELLED, ) action_run_states = [action_run["state"] for action_run in self.client.job_runs(job_run_url)["runs"]] expected = [actionrun.ActionRun.CANCELLED for _ in range(len(action_run_states))] assert_equal(action_run_states, expected) def test_trond_restart_job_with_run_history(self): config = BASIC_CONFIG + textwrap.dedent( """ jobs: - name: fast_job node: local schedule: daily 04:20 actions: - name: single_act command: "sleep 20 && echo good" """ ) self.start_with_config(config) action_run_url = self.client.get_url("MASTER.fast_job.0.single_act") sandbox.wait_on_state( self.client.action_runs, action_run_url, actionrun.ActionRun.RUNNING, ) self.restart_trond() assert_equal( self.client.job_runs(action_run_url)["state"], actionrun.ActionRun.UNKNOWN, ) next_run_url = self.client.get_url("MASTER.fast_job.-1.single_act") sandbox.wait_on_state( self.client.action_runs, next_run_url, actionrun.ActionRun.RUNNING, ) def test_trond_restart_job_running_with_dependencies(self): config = BASIC_CONFIG + textwrap.dedent( """ jobs: - name: complex_job node: local schedule: cron * * * * * actions: - name: first_act command: sleep 20 && echo "I'm waiting" - name: following_act command: echo "thing" requires: ['first_act'] - name: last_act command: echo foo requires: ['following_act'] """ ) self.start_with_config(config) job_name = "MASTER.complex_job" self.sandbox.tronctl("start", job_name) action_run_url = self.client.get_url("MASTER.complex_job.1.first_act") sandbox.wait_on_state( self.client.action_runs, action_run_url, actionrun.ActionRun.RUNNING, ) self.restart_trond() assert_equal( self.client.job_runs(action_run_url)["state"], actionrun.ActionRun.UNKNOWN, ) for followup_action_run in ("following_act", "last_act"): url = self.client.get_url( f"{job_name}.1.{followup_action_run}", ) assert_equal( self.client.action_runs(url)["state"], actionrun.ActionRun.QUEUED, ) ================================================ FILE: tests/trondaemon_test.py ================================================ import os import tempfile import time from unittest import mock from testifycompat import setup from testifycompat import teardown from testifycompat import TestCase from tron.trondaemon import TronDaemon class TronDaemonTestCase(TestCase): @setup @mock.patch("tron.trondaemon.setup_logging", mock.Mock(), autospec=None) @mock.patch("signal.signal", mock.Mock(), autospec=None) def setup(self): self.tmpdir = tempfile.TemporaryDirectory() trond_opts = mock.Mock() trond_opts.working_dir = self.tmpdir.name trond_opts.lock_file = os.path.join(self.tmpdir.name, "lockfile") self.trond = TronDaemon(trond_opts) @teardown def teardown(self): self.tmpdir.cleanup() @mock.patch("tron.trondaemon.setup_logging", mock.Mock(), autospec=None) @mock.patch("signal.signal", mock.Mock(), autospec=None) def test_init(self): daemon = TronDaemon.__new__(TronDaemon) # skip __init__ options = mock.Mock() with mock.patch( "tron.utils.flock", autospec=True, ) as mock_flock: daemon.__init__(options) assert mock_flock.call_count == 0 def test_run_uses_context(self): with mock.patch("tron.trondaemon.setup_logging", mock.Mock(), autospec=None,), mock.patch( "tron.trondaemon.no_daemon_context", mock.Mock(), autospec=None, ) as ndc: ndc.return_value = mock.MagicMock() boot_time = time.time() ndc.return_value.__enter__.side_effect = RuntimeError() daemon = TronDaemon(mock.Mock()) try: daemon.run(boot_time) except Exception: pass assert ndc.call_count == 1 def test_run_manhole_new_manhole(self): with open(self.trond.manhole_sock, "w+"): pass with mock.patch( "twisted.internet.reactor.listenUNIX", autospec=True, ) as mock_listenUNIX: self.trond._run_manhole() assert mock_listenUNIX.call_count == 1 # _run_manhole will remove the old manhole.sock but not recreate # it because we mocked out listenUNIX assert not os.path.exists(self.trond.manhole_sock) ================================================ FILE: tests/utils/__init__.py ================================================ ================================================ FILE: tests/utils/collections_test.py ================================================ from unittest import mock from testifycompat import assert_equal from testifycompat import assert_in from testifycompat import assert_not_in from testifycompat import assert_raises from testifycompat import setup from testifycompat import TestCase from tests.assertions import assert_mock_calls from tests.testingutils import autospec_method from tron.utils import collections class TestMappingCollections(TestCase): @setup def setup_collection(self): self.name = "some_name" self.collection = collections.MappingCollection(self.name) def test_filter_by_name(self): autospec_method(self.collection.remove) self.collection.update(dict.fromkeys(["c", "d", "e"])) self.collection.filter_by_name(["a", "c"]) expected = [mock.call(name) for name in ["d", "e"]] assert_mock_calls(expected, self.collection.remove.mock_calls) def test_remove_missing(self): assert_raises(ValueError, self.collection.remove, "name") def test_remove(self): name = "the_name" self.collection[name] = item = mock.Mock() self.collection.remove(name) assert_not_in(name, self.collection) item.disable.assert_called_with() def test_contains_item_false(self): mock_item, mock_func = mock.Mock(), mock.Mock() assert not self.collection.contains_item(mock_item, mock_func) assert not mock_func.mock_calls def test_contains_item_not_equal(self): mock_item, mock_func = mock.Mock(), mock.Mock() self.collection[mock_item.get_name()] = "other item" result = self.collection.contains_item(mock_item, mock_func) assert_equal(result, mock_func.return_value) mock_func.assert_called_with(mock_item) def test_contains_item_true(self): mock_item, mock_func = mock.Mock(), mock.Mock() self.collection[mock_item.get_name()] = mock_item assert self.collection.contains_item(mock_item, mock_func) def test_add_contains(self): autospec_method(self.collection.contains_item) item, update_func = mock.Mock(), mock.Mock() assert not self.collection.add(item, update_func) assert_not_in(item.get_name(), self.collection) def test_add_new(self): autospec_method(self.collection.contains_item, return_value=False) item, update_func = mock.Mock(), mock.Mock() assert self.collection.add(item, update_func) assert_in(item.get_name(), self.collection) def test_replace(self): autospec_method(self.collection.add) item = mock.Mock() self.collection.replace(item) self.collection.add.assert_called_with( item, self.collection.remove_item, ) ================================================ FILE: tests/utils/crontab_test.py ================================================ from unittest import mock from testifycompat import assert_equal from testifycompat import assert_raises from testifycompat import run from testifycompat import setup from testifycompat import TestCase from tron.utils import crontab class TestConvertPredefined(TestCase): def test_convert_predefined_valid(self): expected = crontab.PREDEFINED_SCHEDULE["@hourly"] assert_equal(crontab.convert_predefined("@hourly"), expected) def test_convert_predefined_invalid(self): assert_raises(ValueError, crontab.convert_predefined, "@bogus") def test_convert_predefined_none(self): line = "something else" assert_equal(crontab.convert_predefined(line), line) class TestParseCrontab(TestCase): def test_parse_asterisk(self): line = "* * * * *" actual = crontab.parse_crontab(line) assert_equal(actual["minutes"], None) assert_equal(actual["hours"], None) assert_equal(actual["months"], None) @mock.patch("tron.utils.crontab.MinuteFieldParser.parse", autospec=True) @mock.patch("tron.utils.crontab.HourFieldParser.parse", autospec=True) @mock.patch("tron.utils.crontab.MonthdayFieldParser.parse", autospec=True) @mock.patch("tron.utils.crontab.MonthFieldParser.parse", autospec=True) @mock.patch("tron.utils.crontab.WeekdayFieldParser.parse", autospec=True) def test_parse(self, mock_dow, mock_month, mock_monthday, mock_hour, mock_min): line = "* * * * *" actual = crontab.parse_crontab(line) assert_equal(actual["minutes"], mock_min.return_value) assert_equal(actual["hours"], mock_hour.return_value) assert_equal(actual["monthdays"], mock_monthday.return_value) assert_equal(actual["months"], mock_month.return_value) assert_equal(actual["weekdays"], mock_dow.return_value) def test_full_crontab_line(self): line = "*/15 0 1,15 * 1-5" expected = { "minutes": [0, 15, 30, 45], "hours": [0], "monthdays": [1, 15], "months": None, "weekdays": [1, 2, 3, 4, 5], "ordinals": None, } assert_equal(crontab.parse_crontab(line), expected) def test_full_crontab_line_with_last(self): line = "0 0 L * *" expected = { "minutes": [0], "hours": [0], "monthdays": ["LAST"], "months": None, "weekdays": None, "ordinals": None, } assert_equal(crontab.parse_crontab(line), expected) class TestMinuteFieldParser(TestCase): @setup def setup_parser(self): self.parser = crontab.MinuteFieldParser() def test_validate_bounds(self): assert_equal(self.parser.validate_bounds(0), 0) assert_equal(self.parser.validate_bounds(59), 59) assert_raises(ValueError, self.parser.validate_bounds, 60) def test_get_values_asterisk(self): assert_equal(self.parser.get_values("*"), list(range(0, 60))) def test_get_values_min_only(self): assert_equal(self.parser.get_values("4"), [4]) assert_equal(self.parser.get_values("33"), [33]) def test_get_values_with_step(self): assert_equal(self.parser.get_values("*/10"), [0, 10, 20, 30, 40, 50]) def test_get_values_with_step_and_range(self): assert_equal(self.parser.get_values("10-30/10"), [10, 20, 30]) def test_get_values_with_step_and_overflow_range(self): assert_equal(self.parser.get_values("30-0/10"), [30, 40, 50, 0]) def test_parse_with_groups(self): assert_equal(self.parser.parse("5,1,7,8,5"), [1, 5, 7, 8]) def test_parse_with_groups_and_ranges(self): expected = [0, 1, 11, 13, 15, 17, 19, 20, 21, 40] assert_equal(self.parser.parse("1,11-22/2,*/20"), expected) class TestMonthFieldParser(TestCase): @setup def setup_parser(self): self.parser = crontab.MonthFieldParser() def test_parse(self): expected = [1, 2, 3, 7, 12] assert_equal(self.parser.parse("DEC, Jan-Feb, jul, MaR"), expected) class TestWeekdayFieldParser(TestCase): @setup def setup_parser(self): self.parser = crontab.WeekdayFieldParser() def test_parser(self): expected = [0, 3, 5, 6] assert_equal(self.parser.parse("Sun, 3, FRI, SaT-Sun"), expected) class TestMonthdayFieldParser(TestCase): @setup def setup_parser(self): self.parser = crontab.MonthdayFieldParser() def test_parse_last(self): expected = [5, 6, "LAST"] assert_equal(self.parser.parse("5, 6, L"), expected) class TestComplexExpressions(TestCase): @setup def setup_parser(self): self.parser = crontab.MinuteFieldParser() def test_complex_expression(self): expected = [0, 10, 20, 30, 40, 50, 55] assert_equal(self.parser.parse("*/10,55"), expected) class TestInvalidInputs(TestCase): @setup def setup_parser(self): self.parser = crontab.MinuteFieldParser() def test_invalid_expression(self): with assert_raises(ValueError): self.parser.parse("61") class TestBoundaryValues(TestCase): @setup def setup_parser(self): self.parser = crontab.MinuteFieldParser() def test_boundary_values(self): assert_equal(self.parser.parse("0"), [0]) assert_equal(self.parser.parse("59"), [59]) if __name__ == "__main__": run() ================================================ FILE: tests/utils/logreader_test.py ================================================ import datetime from unittest import mock import pytest import yaml import tron.utils.logreader from tron.utils.logreader import decompose_action_id from tron.utils.logreader import read_log_stream_for_action_run try: from logreader.readers import S3LogsReader # noqa: F401 except ImportError: pytest.skip("yelp logs readers not available, skipping tests", allow_module_level=True) # used for an explicit patch of staticconf.read return value for an arbitrary namespace def static_conf_patch(args): return lambda arg, namespace, default=None: args.get(arg) def test_read_log_stream_for_action_run_not_available(): with mock.patch("tron.utils.logreader.s3reader_available", False): output = tron.utils.logreader.read_log_stream_for_action_run( "namespace.job.1234.action", component="stdout", min_date=datetime.datetime.now(), max_date=datetime.datetime.now(), paasta_cluster="fake", ) assert "unable to display logs" in output[0] def test_read_log_stream_for_action_run(): with mock.patch( "staticconf.read", autospec=True, side_effect=static_conf_patch({"logging.max_lines_to_display": 1000}), ), mock.patch("tron.config.static_config.build_configuration_watcher", autospec=True,), mock.patch( "tron.config.static_config.load_yaml_file", autospec=True, ), mock.patch( "tron.utils.logreader.get_superregion", autospec=True, return_value="fake" ), mock.patch( "tron.utils.logreader.S3LogsReader", autospec=True ) as mock_s3_reader: mock_s3_reader.return_value.get_log_reader.return_value = iter( [ """{ "tron_run_number": 1234, "component": "stdout", "message": "line 1", "timestamp": "2021-01-02T18:10:09.169421619Z", "cluster": "fake" }""", """{ "tron_run_number": 1234, "component": "stdout", "message": "line 2", "timestamp": "2021-01-02T18:11:09.169421619Z", "cluster": "fake" }""", """{ "tron_run_number": 1234, "component": "stderr", "message": "line 3", "timestamp": "2021-01-02T18:12:09.169421619Z", "cluster": "fake" }""", ] ) output = read_log_stream_for_action_run( "namespace.job.1234.action", component="stdout", min_date=datetime.datetime.now(), max_date=datetime.datetime.now(), paasta_cluster="fake", ) mock_s3_reader.return_value.get_log_reader.assert_called_once_with( log_name="stream_paasta_app_output_namespace_job__action", start_datetime=mock.ANY, end_datetime=mock.ANY ) assert output == ["line 1", "line 2"] @pytest.mark.parametrize( "local_datetime, expected_datetime", [ ( datetime.datetime(2024, 2, 29, 23, 59, 59, tzinfo=datetime.timezone(datetime.timedelta(hours=+3))), datetime.datetime(2024, 2, 29, 20, 59, 59, tzinfo=datetime.timezone.utc), ), ( datetime.datetime(2024, 2, 29, 23, 59, 59, tzinfo=datetime.timezone(datetime.timedelta(hours=-3))), datetime.datetime(2024, 3, 1, 2, 59, 59, tzinfo=datetime.timezone.utc), ), ], ) def test_read_log_stream_for_action_run_tz(local_datetime, expected_datetime): with mock.patch( "staticconf.read", autospec=True, side_effect=static_conf_patch({"logging.max_lines_to_display": 1000}), ), mock.patch("tron.config.static_config.build_configuration_watcher", autospec=True,), mock.patch( "tron.config.static_config.load_yaml_file", autospec=True, ), mock.patch( "tron.utils.logreader.get_superregion", autospec=True, return_value="fake" ), mock.patch( "tron.utils.logreader.S3LogsReader", autospec=True ) as mock_s3_log_reader: read_log_stream_for_action_run( "namespace.job.1234.action", component="stdout", min_date=local_datetime, max_date=local_datetime, paasta_cluster="fake", ) mock_s3_log_reader.return_value.get_log_reader.assert_called_once_with( log_name=mock.ANY, start_datetime=expected_datetime, end_datetime=expected_datetime ) def test_read_log_stream_for_action_run_for_long_output(): # 1000 represents the number of lines that are expected to be # outputted by the test, which is similar to the logging.max_lines_to_display # in tron.yaml in srv-configs max_lines = 1000 with mock.patch("tron.utils.logreader.get_superregion", autospec=True, return_value="fake",), mock.patch( "tron.config.static_config.build_configuration_watcher", autospec=True, ), mock.patch( "staticconf.read", autospec=True, side_effect=static_conf_patch({"logging.max_lines_to_display": 1000}) ), mock.patch( "tron.config.static_config.load_yaml_file", autospec=True, ), mock.patch( "tron.utils.logreader.S3LogsReader", autospec=True ) as mock_s3_reader: with open("./tests/utils/shortOutputTest.txt") as f: content_list = f.readlines() mock_s3_reader.return_value.get_log_reader.return_value = iter(content_list) output = read_log_stream_for_action_run( "namespace.job.228.action", component="stdout", min_date=datetime.datetime.now(), max_date=datetime.datetime.now(), paasta_cluster="infrastage", ) mock_s3_reader.return_value.get_log_reader.assert_called_once_with( log_name="stream_paasta_app_output_namespace_job__action", start_datetime=mock.ANY, end_datetime=mock.ANY ) assert len(output) == max_lines + 1 def test_decompose_action_id_file_not_found(): action_run_id = "namespace.job.1234.action" paasta_cluster = "fake_cluster" with mock.patch("builtins.open", side_effect=FileNotFoundError): namespace, job_name, run_num, action = decompose_action_id(action_run_id, paasta_cluster) assert namespace == "namespace" assert job_name == "job" assert run_num == "1234" assert action == "action" def test_decompose_action_id_yaml_error(): action_run_id = "namespace.job.1234.action" paasta_cluster = "fake_cluster" with mock.patch("builtins.open", mock.mock_open(read_data="invalid_yaml")), mock.patch( "yaml.safe_load", side_effect=yaml.YAMLError ): namespace, job_name, run_num, action = decompose_action_id(action_run_id, paasta_cluster) assert namespace == "namespace" assert job_name == "job" assert run_num == "1234" assert action == "action" def test_decompose_action_id_generic_error(): action_run_id = "namespace.job.1234.action" paasta_cluster = "fake_cluster" with mock.patch("builtins.open", mock.mock_open(read_data="some_data")), mock.patch( "yaml.safe_load", side_effect=Exception ): namespace, job_name, run_num, action = decompose_action_id(action_run_id, paasta_cluster) assert namespace == "namespace" assert job_name == "job" assert run_num == "1234" assert action == "action" def test_decompose_action_id_service_not_found(): action_run_id = "namespace.job.1234.action" paasta_cluster = "fake_cluster" config_content = """ job: actions: action: command: "sleep 10" """ with mock.patch("builtins.open", mock.mock_open(read_data=config_content)), mock.patch( "yaml.safe_load", return_value=yaml.safe_load(config_content) ): namespace, job_name, run_num, action = decompose_action_id(action_run_id, paasta_cluster) assert namespace == "namespace" assert job_name == "job" assert run_num == "1234" assert action == "action" ================================================ FILE: tests/utils/observer_test.py ================================================ from unittest import mock from testifycompat import assert_equal from testifycompat import run from testifycompat import setup from testifycompat import TestCase from tests.assertions import assert_length from tron.utils.observer import Observable from tron.utils.observer import Observer class TestObservable(TestCase): @setup def setup_observer(self): self.obs = Observable() def test_attach(self): def func(): return 1 self.obs.attach("a", func) assert_equal(len(self.obs._observers), 1) assert_equal(self.obs._observers["a"], [func]) def test_listen_seq(self): def func(): return 1 self.obs.attach(["a", "b"], func) assert_equal(len(self.obs._observers), 2) assert_equal(self.obs._observers["a"], [func]) assert_equal(self.obs._observers["b"], [func]) def test_notify(self): handler = mock.MagicMock() self.obs.attach(["a", "b"], handler) self.obs.notify("a") assert_equal(len(handler.handler.mock_calls), 1) self.obs.notify("b") assert_equal(len(handler.handler.mock_calls), 2) class TestObserverClear(TestCase): @setup def setup_observer(self): self.obs = Observable() def func(): return 1 self.obs.attach("a", func) self.obs.attach("b", func) self.obs.attach(True, func) self.obs.attach(["a", "b"], func) def test_clear_listeners_all(self): self.obs.clear_observers() assert_equal(len(self.obs._observers), 0) def test_clear_listeners_some(self): self.obs.clear_observers("a") assert_equal(len(self.obs._observers), 2) assert_equal(set(self.obs._observers.keys()), {True, "b"}) def test_remove_observer_none(self): def observer(): return 2 self.obs.remove_observer(observer) assert_equal(set(self.obs._observers.keys()), {True, "a", "b"}) assert_length(self.obs._observers["a"], 2) assert_length(self.obs._observers["b"], 2) assert_length(self.obs._observers[True], 1) def test_remove_observer(self): def observer(): return 2 self.obs.attach("a", observer) self.obs.attach("c", observer) self.obs.remove_observer(observer) assert_length(self.obs._observers["a"], 2) assert_length(self.obs._observers["b"], 2) assert_length(self.obs._observers[True], 1) assert_length(self.obs._observers["c"], 0) class MockObserver(Observer): def __init__(self, obs, event): self.obs = obs self.event = event self.watch(obs, event) self.has_watched = 0 self.event_data = None def handler(self, obs, event, event_data): assert_equal(obs, self.obs) assert_equal(event, self.event) self.has_watched += 1 self.event_data = event_data class TestObserver(TestCase): @setup def setup_observer(self): self.obs = Observable() def test_watch(self): event = "FIVE" handler = MockObserver(self.obs, event) self.obs.notify(event) assert_equal(handler.has_watched, 1) assert handler.event_data is None self.obs.notify("other event") assert_equal(handler.has_watched, 1) self.obs.notify(event, "event_data") assert_equal(handler.has_watched, 2) assert handler.event_data == "event_data" if __name__ == "__main__": run() ================================================ FILE: tests/utils/proxy_test.py ================================================ from testifycompat import assert_equal from testifycompat import assert_in from testifycompat import assert_raises from testifycompat import run from testifycompat import setup from testifycompat import TestCase from tron.utils.proxy import AttributeProxy from tron.utils.proxy import CollectionProxy class DummyTarget: def __init__(self, v): self.v = v def foo(self): return self.v @property def not_foo(self): return not self.v def equals(self, b, sometimes=False): if sometimes: return "sometimes" return self.v == b class DummyObject: def __init__(self, proxy): self.proxy = proxy def __getattr__(self, item): return self.proxy.perform(item) class TestCollectionProxy(TestCase): @setup def setup_proxy(self): self.target_list = [DummyTarget(1), DummyTarget(2), DummyTarget(0)] self.proxy = CollectionProxy( lambda: self.target_list, [ ("foo", any, True), ("not_foo", all, False), ("equals", lambda a: list(a), True), ], ) self.dummy = DummyObject(self.proxy) def test_add(self): self.proxy.add("foo", any, True) assert_equal(self.proxy._defs["foo"], (any, True)) def test_perform(self): assert self.dummy.foo() assert not self.dummy.not_foo def test_perform_not_defined(self): assert_raises(AttributeError, self.dummy.proxy.perform, "bar") def test_perform_with_params(self): assert_equal(self.proxy.perform("equals")(2), [False, True, False]) sometimes = ["sometimes"] * 3 assert_equal( self.proxy.perform("equals")(3, sometimes=True), sometimes, ) class TestAttributeProxy(TestCase): @setup def setup_proxy(self): self.target = DummyTarget(1) self.proxy = AttributeProxy(self.target, ["foo", "not_foo"]) self.dummy = DummyObject(self.proxy) def test_add(self): self.proxy.add("bar") assert_in("bar", self.proxy._attributes) def test_perform(self): assert_equal(self.dummy.foo(), 1) assert_equal(self.dummy.not_foo, False) def test_perform_not_defined(self): assert_raises(AttributeError, self.dummy.proxy.perform, "zzz") if __name__ == "__main__": run() ================================================ FILE: tests/utils/shortOutputTest.txt ================================================ {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1","timestamp":"2022-08-12T13:30:04.686637906Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"5","timestamp":"2022-08-12T13:30:04.686730242Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"2","timestamp":"2022-08-12T13:30:04.686653695Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"3","timestamp":"2022-08-12T13:30:04.68670797Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"6","timestamp":"2022-08-12T13:30:04.686799509Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"4","timestamp":"2022-08-12T13:30:04.686710726Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"7","timestamp":"2022-08-12T13:30:04.68702548Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"8","timestamp":"2022-08-12T13:30:04.687027597Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"9","timestamp":"2022-08-12T13:30:04.687077736Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"10","timestamp":"2022-08-12T13:30:04.687161226Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"11","timestamp":"2022-08-12T13:30:04.687273728Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"12","timestamp":"2022-08-12T13:30:04.687277071Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"13","timestamp":"2022-08-12T13:30:04.687306184Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"14","timestamp":"2022-08-12T13:30:04.687341593Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"15","timestamp":"2022-08-12T13:30:04.687488029Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"16","timestamp":"2022-08-12T13:30:04.687489495Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"17","timestamp":"2022-08-12T13:30:04.687502175Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"18","timestamp":"2022-08-12T13:30:04.687542498Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"21","timestamp":"2022-08-12T13:30:04.687638635Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"19","timestamp":"2022-08-12T13:30:04.687618198Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"22","timestamp":"2022-08-12T13:30:04.687670676Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"20","timestamp":"2022-08-12T13:30:04.687620336Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"23","timestamp":"2022-08-12T13:30:04.687813739Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"24","timestamp":"2022-08-12T13:30:04.687817306Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"25","timestamp":"2022-08-12T13:30:04.687839161Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"26","timestamp":"2022-08-12T13:30:04.687878011Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"27","timestamp":"2022-08-12T13:30:04.688001387Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"28","timestamp":"2022-08-12T13:30:04.688003044Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"29","timestamp":"2022-08-12T13:30:04.688014036Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"30","timestamp":"2022-08-12T13:30:04.688051272Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"31","timestamp":"2022-08-12T13:30:04.688126709Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"32","timestamp":"2022-08-12T13:30:04.688128322Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"33","timestamp":"2022-08-12T13:30:04.688139583Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"34","timestamp":"2022-08-12T13:30:04.68816411Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"35","timestamp":"2022-08-12T13:30:04.688234038Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"36","timestamp":"2022-08-12T13:30:04.688235549Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"37","timestamp":"2022-08-12T13:30:04.68824796Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"38","timestamp":"2022-08-12T13:30:04.688278753Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"39","timestamp":"2022-08-12T13:30:04.688332263Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"40","timestamp":"2022-08-12T13:30:04.688333783Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"41","timestamp":"2022-08-12T13:30:04.688349331Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"42","timestamp":"2022-08-12T13:30:04.688377769Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"43","timestamp":"2022-08-12T13:30:04.688437161Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"44","timestamp":"2022-08-12T13:30:04.688439485Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"45","timestamp":"2022-08-12T13:30:04.688456515Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"46","timestamp":"2022-08-12T13:30:04.68848179Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"47","timestamp":"2022-08-12T13:30:04.688559693Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"48","timestamp":"2022-08-12T13:30:04.688561281Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"53","timestamp":"2022-08-12T13:30:04.688689148Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"54","timestamp":"2022-08-12T13:30:04.688714557Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"55","timestamp":"2022-08-12T13:30:04.688860562Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"49","timestamp":"2022-08-12T13:30:04.688573177Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"56","timestamp":"2022-08-12T13:30:04.68886246Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"50","timestamp":"2022-08-12T13:30:04.688606432Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"51","timestamp":"2022-08-12T13:30:04.688673085Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"52","timestamp":"2022-08-12T13:30:04.688675724Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"57","timestamp":"2022-08-12T13:30:04.688875227Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"58","timestamp":"2022-08-12T13:30:04.688930281Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"59","timestamp":"2022-08-12T13:30:04.688975956Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"60","timestamp":"2022-08-12T13:30:04.688986533Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"61","timestamp":"2022-08-12T13:30:04.688994591Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"62","timestamp":"2022-08-12T13:30:04.689021591Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"63","timestamp":"2022-08-12T13:30:04.689171143Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"64","timestamp":"2022-08-12T13:30:04.689173043Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"65","timestamp":"2022-08-12T13:30:04.68918868Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"66","timestamp":"2022-08-12T13:30:04.689231633Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"67","timestamp":"2022-08-12T13:30:04.689332791Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"68","timestamp":"2022-08-12T13:30:04.689334352Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"69","timestamp":"2022-08-12T13:30:04.689345198Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"70","timestamp":"2022-08-12T13:30:04.68937079Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"71","timestamp":"2022-08-12T13:30:04.689463921Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"77","timestamp":"2022-08-12T13:30:04.689602894Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"72","timestamp":"2022-08-12T13:30:04.689466356Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"78","timestamp":"2022-08-12T13:30:04.68963501Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"79","timestamp":"2022-08-12T13:30:04.689700114Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"80","timestamp":"2022-08-12T13:30:04.689702805Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"73","timestamp":"2022-08-12T13:30:04.689484797Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"74","timestamp":"2022-08-12T13:30:04.689509529Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"75","timestamp":"2022-08-12T13:30:04.689590003Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"76","timestamp":"2022-08-12T13:30:04.689591492Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"85","timestamp":"2022-08-12T13:30:04.689884469Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"81","timestamp":"2022-08-12T13:30:04.689718471Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"86","timestamp":"2022-08-12T13:30:04.689930036Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"82","timestamp":"2022-08-12T13:30:04.689755218Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"87","timestamp":"2022-08-12T13:30:04.690078096Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"83","timestamp":"2022-08-12T13:30:04.689868169Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"89","timestamp":"2022-08-12T13:30:04.690121194Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"88","timestamp":"2022-08-12T13:30:04.690081266Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"84","timestamp":"2022-08-12T13:30:04.689870538Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"90","timestamp":"2022-08-12T13:30:04.690123522Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"93","timestamp":"2022-08-12T13:30:04.690276406Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"91","timestamp":"2022-08-12T13:30:04.690263137Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"92","timestamp":"2022-08-12T13:30:04.690264843Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"97","timestamp":"2022-08-12T13:30:04.690407214Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"101","timestamp":"2022-08-12T13:30:04.690536078Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"98","timestamp":"2022-08-12T13:30:04.690433197Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"102","timestamp":"2022-08-12T13:30:04.690560435Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"99","timestamp":"2022-08-12T13:30:04.690522808Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"103","timestamp":"2022-08-12T13:30:04.690633135Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"100","timestamp":"2022-08-12T13:30:04.690524308Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"104","timestamp":"2022-08-12T13:30:04.690634614Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"94","timestamp":"2022-08-12T13:30:04.690304932Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"95","timestamp":"2022-08-12T13:30:04.690392724Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"96","timestamp":"2022-08-12T13:30:04.690394266Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"105","timestamp":"2022-08-12T13:30:04.69064582Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"106","timestamp":"2022-08-12T13:30:04.690669619Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"107","timestamp":"2022-08-12T13:30:04.690753237Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"108","timestamp":"2022-08-12T13:30:04.690755122Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"109","timestamp":"2022-08-12T13:30:04.690772753Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"110","timestamp":"2022-08-12T13:30:04.69080219Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"111","timestamp":"2022-08-12T13:30:04.690877097Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"112","timestamp":"2022-08-12T13:30:04.690878543Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"113","timestamp":"2022-08-12T13:30:04.690893601Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"114","timestamp":"2022-08-12T13:30:04.690919509Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"117","timestamp":"2022-08-12T13:30:04.691041068Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"118","timestamp":"2022-08-12T13:30:04.691063975Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"119","timestamp":"2022-08-12T13:30:04.691140054Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"129","timestamp":"2022-08-12T13:30:04.691387414Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"120","timestamp":"2022-08-12T13:30:04.691141565Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"130","timestamp":"2022-08-12T13:30:04.691410889Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"131","timestamp":"2022-08-12T13:30:04.691497238Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"132","timestamp":"2022-08-12T13:30:04.691498657Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"125","timestamp":"2022-08-12T13:30:04.691282187Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"126","timestamp":"2022-08-12T13:30:04.691305594Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"121","timestamp":"2022-08-12T13:30:04.691153235Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"127","timestamp":"2022-08-12T13:30:04.691372941Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"122","timestamp":"2022-08-12T13:30:04.691176001Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"128","timestamp":"2022-08-12T13:30:04.69137442Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"123","timestamp":"2022-08-12T13:30:04.691270038Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"124","timestamp":"2022-08-12T13:30:04.691271381Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"115","timestamp":"2022-08-12T13:30:04.691026241Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"116","timestamp":"2022-08-12T13:30:04.691028254Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"133","timestamp":"2022-08-12T13:30:04.691509865Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"134","timestamp":"2022-08-12T13:30:04.691531912Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"135","timestamp":"2022-08-12T13:30:04.691603332Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"136","timestamp":"2022-08-12T13:30:04.691604736Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"137","timestamp":"2022-08-12T13:30:04.691618847Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"138","timestamp":"2022-08-12T13:30:04.691642663Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"139","timestamp":"2022-08-12T13:30:04.691731628Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"140","timestamp":"2022-08-12T13:30:04.691732919Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"141","timestamp":"2022-08-12T13:30:04.691743131Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"142","timestamp":"2022-08-12T13:30:04.691765184Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"143","timestamp":"2022-08-12T13:30:04.691832098Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"144","timestamp":"2022-08-12T13:30:04.691834198Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"145","timestamp":"2022-08-12T13:30:04.691845287Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"146","timestamp":"2022-08-12T13:30:04.69187195Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"147","timestamp":"2022-08-12T13:30:04.691956577Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"148","timestamp":"2022-08-12T13:30:04.691958009Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"149","timestamp":"2022-08-12T13:30:04.691969147Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"150","timestamp":"2022-08-12T13:30:04.69199273Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"151","timestamp":"2022-08-12T13:30:04.692062796Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"152","timestamp":"2022-08-12T13:30:04.692064213Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"153","timestamp":"2022-08-12T13:30:04.692076476Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"157","timestamp":"2022-08-12T13:30:04.692195395Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"154","timestamp":"2022-08-12T13:30:04.692100205Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"158","timestamp":"2022-08-12T13:30:04.692241973Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"155","timestamp":"2022-08-12T13:30:04.692178771Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"159","timestamp":"2022-08-12T13:30:04.692298151Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"156","timestamp":"2022-08-12T13:30:04.692180225Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"160","timestamp":"2022-08-12T13:30:04.692301626Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"161","timestamp":"2022-08-12T13:30:04.692317326Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"176","timestamp":"2022-08-12T13:30:04.692875814Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"162","timestamp":"2022-08-12T13:30:04.692345913Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"172","timestamp":"2022-08-12T13:30:04.692728279Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"163","timestamp":"2022-08-12T13:30:04.69242628Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"173","timestamp":"2022-08-12T13:30:04.692741323Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"164","timestamp":"2022-08-12T13:30:04.692427856Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"174","timestamp":"2022-08-12T13:30:04.692862163Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"175","timestamp":"2022-08-12T13:30:04.692863706Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"169","timestamp":"2022-08-12T13:30:04.69266267Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"170","timestamp":"2022-08-12T13:30:04.692677926Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"171","timestamp":"2022-08-12T13:30:04.692726759Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"165","timestamp":"2022-08-12T13:30:04.69243928Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"166","timestamp":"2022-08-12T13:30:04.692466086Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"167","timestamp":"2022-08-12T13:30:04.692537679Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"177","timestamp":"2022-08-12T13:30:04.692907917Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"168","timestamp":"2022-08-12T13:30:04.692572346Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"178","timestamp":"2022-08-12T13:30:04.692983835Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"179","timestamp":"2022-08-12T13:30:04.692985498Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"180","timestamp":"2022-08-12T13:30:04.69299595Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"181","timestamp":"2022-08-12T13:30:04.693022057Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"182","timestamp":"2022-08-12T13:30:04.693098237Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"184","timestamp":"2022-08-12T13:30:04.693111014Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"183","timestamp":"2022-08-12T13:30:04.693100161Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"192","timestamp":"2022-08-12T13:30:04.693415579Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"193","timestamp":"2022-08-12T13:30:04.693417373Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"196","timestamp":"2022-08-12T13:30:04.693524935Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"194","timestamp":"2022-08-12T13:30:04.693510485Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"197","timestamp":"2022-08-12T13:30:04.693553832Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"195","timestamp":"2022-08-12T13:30:04.693512123Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"198","timestamp":"2022-08-12T13:30:04.693635481Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"199","timestamp":"2022-08-12T13:30:04.693637098Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"188","timestamp":"2022-08-12T13:30:04.693243745Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"189","timestamp":"2022-08-12T13:30:04.69328921Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"190","timestamp":"2022-08-12T13:30:04.693356023Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"185","timestamp":"2022-08-12T13:30:04.693141543Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"191","timestamp":"2022-08-12T13:30:04.693357893Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"186","timestamp":"2022-08-12T13:30:04.693224619Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"200","timestamp":"2022-08-12T13:30:04.693648973Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"187","timestamp":"2022-08-12T13:30:04.693226683Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"201","timestamp":"2022-08-12T13:30:04.693679566Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"202","timestamp":"2022-08-12T13:30:04.693749798Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"203","timestamp":"2022-08-12T13:30:04.69375125Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"204","timestamp":"2022-08-12T13:30:04.693762378Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"205","timestamp":"2022-08-12T13:30:04.693787008Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"206","timestamp":"2022-08-12T13:30:04.693847183Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"208","timestamp":"2022-08-12T13:30:04.693859593Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"209","timestamp":"2022-08-12T13:30:04.693885136Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"216","timestamp":"2022-08-12T13:30:04.694060214Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"210","timestamp":"2022-08-12T13:30:04.693945469Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"217","timestamp":"2022-08-12T13:30:04.694083789Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"211","timestamp":"2022-08-12T13:30:04.693947282Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"218","timestamp":"2022-08-12T13:30:04.694145516Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"219","timestamp":"2022-08-12T13:30:04.694146949Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"212","timestamp":"2022-08-12T13:30:04.693963187Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"213","timestamp":"2022-08-12T13:30:04.693987556Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"214","timestamp":"2022-08-12T13:30:04.694046409Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"207","timestamp":"2022-08-12T13:30:04.693848965Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"215","timestamp":"2022-08-12T13:30:04.694047924Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"220","timestamp":"2022-08-12T13:30:04.694158589Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"221","timestamp":"2022-08-12T13:30:04.69418191Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"222","timestamp":"2022-08-12T13:30:04.694253672Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"223","timestamp":"2022-08-12T13:30:04.694255467Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"224","timestamp":"2022-08-12T13:30:04.69426637Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"225","timestamp":"2022-08-12T13:30:04.694292993Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"226","timestamp":"2022-08-12T13:30:04.694349649Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"228","timestamp":"2022-08-12T13:30:04.694361903Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"227","timestamp":"2022-08-12T13:30:04.694351011Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"229","timestamp":"2022-08-12T13:30:04.694384807Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"230","timestamp":"2022-08-12T13:30:04.694447415Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"231","timestamp":"2022-08-12T13:30:04.69444895Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"240","timestamp":"2022-08-12T13:30:04.69465744Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"241","timestamp":"2022-08-12T13:30:04.694683152Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"236","timestamp":"2022-08-12T13:30:04.694558979Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"242","timestamp":"2022-08-12T13:30:04.694743634Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"237","timestamp":"2022-08-12T13:30:04.69458339Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"243","timestamp":"2022-08-12T13:30:04.694744993Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"238","timestamp":"2022-08-12T13:30:04.694644799Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"239","timestamp":"2022-08-12T13:30:04.694646165Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"232","timestamp":"2022-08-12T13:30:04.694460741Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"233","timestamp":"2022-08-12T13:30:04.694484631Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"234","timestamp":"2022-08-12T13:30:04.6945459Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"235","timestamp":"2022-08-12T13:30:04.694547231Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"244","timestamp":"2022-08-12T13:30:04.694757562Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"245","timestamp":"2022-08-12T13:30:04.694781236Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"246","timestamp":"2022-08-12T13:30:04.694842613Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"247","timestamp":"2022-08-12T13:30:04.694844304Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"248","timestamp":"2022-08-12T13:30:04.694855437Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"249","timestamp":"2022-08-12T13:30:04.694879208Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"250","timestamp":"2022-08-12T13:30:04.695113263Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"251","timestamp":"2022-08-12T13:30:04.695114808Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"252","timestamp":"2022-08-12T13:30:04.695126598Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"253","timestamp":"2022-08-12T13:30:04.695153021Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"254","timestamp":"2022-08-12T13:30:04.695215823Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"260","timestamp":"2022-08-12T13:30:04.695334059Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"255","timestamp":"2022-08-12T13:30:04.695217257Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"261","timestamp":"2022-08-12T13:30:04.69535803Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"262","timestamp":"2022-08-12T13:30:04.695419545Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"256","timestamp":"2022-08-12T13:30:04.695228423Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"263","timestamp":"2022-08-12T13:30:04.695420919Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"257","timestamp":"2022-08-12T13:30:04.695252559Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"258","timestamp":"2022-08-12T13:30:04.695322097Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"259","timestamp":"2022-08-12T13:30:04.695323489Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"264","timestamp":"2022-08-12T13:30:04.695432895Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"265","timestamp":"2022-08-12T13:30:04.69545634Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"266","timestamp":"2022-08-12T13:30:04.695518181Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"268","timestamp":"2022-08-12T13:30:04.695531492Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"267","timestamp":"2022-08-12T13:30:04.69551958Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"269","timestamp":"2022-08-12T13:30:04.695555392Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"270","timestamp":"2022-08-12T13:30:04.695618372Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"271","timestamp":"2022-08-12T13:30:04.695619772Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"272","timestamp":"2022-08-12T13:30:04.695630522Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"273","timestamp":"2022-08-12T13:30:04.695655147Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"274","timestamp":"2022-08-12T13:30:04.695714559Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"275","timestamp":"2022-08-12T13:30:04.695715969Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"280","timestamp":"2022-08-12T13:30:04.695825707Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"281","timestamp":"2022-08-12T13:30:04.695849009Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"282","timestamp":"2022-08-12T13:30:04.695911493Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"276","timestamp":"2022-08-12T13:30:04.695728339Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"283","timestamp":"2022-08-12T13:30:04.695912958Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"277","timestamp":"2022-08-12T13:30:04.695751608Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"278","timestamp":"2022-08-12T13:30:04.695814162Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"279","timestamp":"2022-08-12T13:30:04.69581555Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"284","timestamp":"2022-08-12T13:30:04.695924267Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"285","timestamp":"2022-08-12T13:30:04.695947645Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"286","timestamp":"2022-08-12T13:30:04.696008189Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"287","timestamp":"2022-08-12T13:30:04.696009573Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"288","timestamp":"2022-08-12T13:30:04.696020269Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"289","timestamp":"2022-08-12T13:30:04.69604242Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"290","timestamp":"2022-08-12T13:30:04.696093545Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"291","timestamp":"2022-08-12T13:30:04.696095061Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"292","timestamp":"2022-08-12T13:30:04.696105202Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"293","timestamp":"2022-08-12T13:30:04.69612826Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"294","timestamp":"2022-08-12T13:30:04.696195685Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"296","timestamp":"2022-08-12T13:30:04.696227173Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"295","timestamp":"2022-08-12T13:30:04.696197817Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"297","timestamp":"2022-08-12T13:30:04.696242784Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"298","timestamp":"2022-08-12T13:30:04.69638203Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"299","timestamp":"2022-08-12T13:30:04.69638378Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"300","timestamp":"2022-08-12T13:30:04.696398687Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"301","timestamp":"2022-08-12T13:30:04.696430352Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"302","timestamp":"2022-08-12T13:30:04.696518311Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"303","timestamp":"2022-08-12T13:30:04.69652062Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"304","timestamp":"2022-08-12T13:30:04.696540129Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"305","timestamp":"2022-08-12T13:30:04.696590149Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"306","timestamp":"2022-08-12T13:30:04.696660475Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"307","timestamp":"2022-08-12T13:30:04.696662012Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"308","timestamp":"2022-08-12T13:30:04.696690796Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"309","timestamp":"2022-08-12T13:30:04.696730877Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"310","timestamp":"2022-08-12T13:30:04.696830771Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"311","timestamp":"2022-08-12T13:30:04.696832305Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"312","timestamp":"2022-08-12T13:30:04.696845426Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"313","timestamp":"2022-08-12T13:30:04.696873105Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"314","timestamp":"2022-08-12T13:30:04.69695739Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"315","timestamp":"2022-08-12T13:30:04.696958961Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"316","timestamp":"2022-08-12T13:30:04.696972666Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"317","timestamp":"2022-08-12T13:30:04.697001155Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"318","timestamp":"2022-08-12T13:30:04.697072945Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"319","timestamp":"2022-08-12T13:30:04.697074408Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"320","timestamp":"2022-08-12T13:30:04.697085567Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"321","timestamp":"2022-08-12T13:30:04.697108718Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"322","timestamp":"2022-08-12T13:30:04.697166986Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"323","timestamp":"2022-08-12T13:30:04.697168337Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"324","timestamp":"2022-08-12T13:30:04.697180091Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"325","timestamp":"2022-08-12T13:30:04.697202922Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"326","timestamp":"2022-08-12T13:30:04.697268647Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"327","timestamp":"2022-08-12T13:30:04.697269994Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"328","timestamp":"2022-08-12T13:30:04.697281008Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"329","timestamp":"2022-08-12T13:30:04.697304114Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"330","timestamp":"2022-08-12T13:30:04.69735311Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"332","timestamp":"2022-08-12T13:30:04.697364732Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"331","timestamp":"2022-08-12T13:30:04.697354421Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"333","timestamp":"2022-08-12T13:30:04.697387362Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"334","timestamp":"2022-08-12T13:30:04.697464404Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"335","timestamp":"2022-08-12T13:30:04.69746577Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"336","timestamp":"2022-08-12T13:30:04.697476992Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"337","timestamp":"2022-08-12T13:30:04.697499001Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"338","timestamp":"2022-08-12T13:30:04.697548663Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"339","timestamp":"2022-08-12T13:30:04.697550048Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"340","timestamp":"2022-08-12T13:30:04.697560534Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"341","timestamp":"2022-08-12T13:30:04.69758234Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"342","timestamp":"2022-08-12T13:30:04.697637478Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"343","timestamp":"2022-08-12T13:30:04.697638783Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"344","timestamp":"2022-08-12T13:30:04.697652488Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"345","timestamp":"2022-08-12T13:30:04.69767455Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"346","timestamp":"2022-08-12T13:30:04.697733548Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"347","timestamp":"2022-08-12T13:30:04.697734904Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"348","timestamp":"2022-08-12T13:30:04.697749361Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"349","timestamp":"2022-08-12T13:30:04.697774665Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"350","timestamp":"2022-08-12T13:30:04.697829909Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"351","timestamp":"2022-08-12T13:30:04.697831278Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"352","timestamp":"2022-08-12T13:30:04.697843048Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"353","timestamp":"2022-08-12T13:30:04.697865318Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"354","timestamp":"2022-08-12T13:30:04.697914563Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"355","timestamp":"2022-08-12T13:30:04.697915969Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"356","timestamp":"2022-08-12T13:30:04.697926155Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"357","timestamp":"2022-08-12T13:30:04.697949141Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"358","timestamp":"2022-08-12T13:30:04.697999614Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"359","timestamp":"2022-08-12T13:30:04.698000967Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"360","timestamp":"2022-08-12T13:30:04.698010865Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"361","timestamp":"2022-08-12T13:30:04.698033605Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"362","timestamp":"2022-08-12T13:30:04.698083348Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"363","timestamp":"2022-08-12T13:30:04.698084693Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"364","timestamp":"2022-08-12T13:30:04.698095857Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"365","timestamp":"2022-08-12T13:30:04.698117936Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"366","timestamp":"2022-08-12T13:30:04.698168574Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"367","timestamp":"2022-08-12T13:30:04.698169972Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"368","timestamp":"2022-08-12T13:30:04.698180376Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"369","timestamp":"2022-08-12T13:30:04.698201946Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"370","timestamp":"2022-08-12T13:30:04.698254245Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"371","timestamp":"2022-08-12T13:30:04.698255635Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"372","timestamp":"2022-08-12T13:30:04.698266316Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"373","timestamp":"2022-08-12T13:30:04.698287594Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"374","timestamp":"2022-08-12T13:30:04.698339099Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"375","timestamp":"2022-08-12T13:30:04.698340447Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"376","timestamp":"2022-08-12T13:30:04.698352046Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"377","timestamp":"2022-08-12T13:30:04.698373786Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"378","timestamp":"2022-08-12T13:30:04.698425574Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"379","timestamp":"2022-08-12T13:30:04.698426889Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"380","timestamp":"2022-08-12T13:30:04.698438744Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"381","timestamp":"2022-08-12T13:30:04.698461298Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"382","timestamp":"2022-08-12T13:30:04.698527063Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"383","timestamp":"2022-08-12T13:30:04.698528604Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"384","timestamp":"2022-08-12T13:30:04.698539973Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"385","timestamp":"2022-08-12T13:30:04.698562009Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"386","timestamp":"2022-08-12T13:30:04.698610715Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"387","timestamp":"2022-08-12T13:30:04.698612126Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"388","timestamp":"2022-08-12T13:30:04.698623135Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"389","timestamp":"2022-08-12T13:30:04.698645389Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"390","timestamp":"2022-08-12T13:30:04.698697102Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"391","timestamp":"2022-08-12T13:30:04.698698497Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"392","timestamp":"2022-08-12T13:30:04.698709277Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"393","timestamp":"2022-08-12T13:30:04.698731161Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"394","timestamp":"2022-08-12T13:30:04.698783297Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"395","timestamp":"2022-08-12T13:30:04.698784691Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"396","timestamp":"2022-08-12T13:30:04.698795352Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"397","timestamp":"2022-08-12T13:30:04.698817557Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"398","timestamp":"2022-08-12T13:30:04.698869718Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"399","timestamp":"2022-08-12T13:30:04.698871062Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"400","timestamp":"2022-08-12T13:30:04.698881666Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"401","timestamp":"2022-08-12T13:30:04.698902766Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"402","timestamp":"2022-08-12T13:30:04.698954504Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"403","timestamp":"2022-08-12T13:30:04.698955895Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"404","timestamp":"2022-08-12T13:30:04.698966515Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"405","timestamp":"2022-08-12T13:30:04.698988908Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"406","timestamp":"2022-08-12T13:30:04.699042398Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"407","timestamp":"2022-08-12T13:30:04.699043801Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"408","timestamp":"2022-08-12T13:30:04.699055661Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"409","timestamp":"2022-08-12T13:30:04.699107001Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"410","timestamp":"2022-08-12T13:30:04.699186359Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"411","timestamp":"2022-08-12T13:30:04.699197581Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"412","timestamp":"2022-08-12T13:30:04.699205971Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"413","timestamp":"2022-08-12T13:30:04.699217868Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"414","timestamp":"2022-08-12T13:30:04.699302302Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"415","timestamp":"2022-08-12T13:30:04.699304503Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"416","timestamp":"2022-08-12T13:30:04.699317968Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"417","timestamp":"2022-08-12T13:30:04.699346817Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"418","timestamp":"2022-08-12T13:30:04.699428241Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"419","timestamp":"2022-08-12T13:30:04.69942974Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"420","timestamp":"2022-08-12T13:30:04.699441257Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"421","timestamp":"2022-08-12T13:30:04.699464746Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"422","timestamp":"2022-08-12T13:30:04.699548025Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"423","timestamp":"2022-08-12T13:30:04.699549752Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"424","timestamp":"2022-08-12T13:30:04.699560608Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"425","timestamp":"2022-08-12T13:30:04.699584562Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"426","timestamp":"2022-08-12T13:30:04.699672833Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"427","timestamp":"2022-08-12T13:30:04.699674218Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"428","timestamp":"2022-08-12T13:30:04.699686007Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"429","timestamp":"2022-08-12T13:30:04.699709147Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"430","timestamp":"2022-08-12T13:30:04.699790251Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"431","timestamp":"2022-08-12T13:30:04.69979235Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"432","timestamp":"2022-08-12T13:30:04.699809916Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"433","timestamp":"2022-08-12T13:30:04.699856973Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"434","timestamp":"2022-08-12T13:30:04.699917961Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"435","timestamp":"2022-08-12T13:30:04.699925398Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"436","timestamp":"2022-08-12T13:30:04.699959003Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"437","timestamp":"2022-08-12T13:30:04.699960449Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"438","timestamp":"2022-08-12T13:30:04.700031357Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"439","timestamp":"2022-08-12T13:30:04.700032901Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"440","timestamp":"2022-08-12T13:30:04.700045687Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"441","timestamp":"2022-08-12T13:30:04.700070031Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"442","timestamp":"2022-08-12T13:30:04.700146267Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"443","timestamp":"2022-08-12T13:30:04.700147735Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"444","timestamp":"2022-08-12T13:30:04.7001586Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"445","timestamp":"2022-08-12T13:30:04.70018244Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"446","timestamp":"2022-08-12T13:30:04.700243742Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"447","timestamp":"2022-08-12T13:30:04.700245134Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"448","timestamp":"2022-08-12T13:30:04.700257598Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"449","timestamp":"2022-08-12T13:30:04.700280541Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"450","timestamp":"2022-08-12T13:30:04.700343448Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"451","timestamp":"2022-08-12T13:30:04.700345049Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"452","timestamp":"2022-08-12T13:30:04.700357818Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"453","timestamp":"2022-08-12T13:30:04.700382802Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"454","timestamp":"2022-08-12T13:30:04.700450589Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"455","timestamp":"2022-08-12T13:30:04.700452059Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"456","timestamp":"2022-08-12T13:30:04.700466067Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"457","timestamp":"2022-08-12T13:30:04.700490112Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"458","timestamp":"2022-08-12T13:30:04.700553243Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"459","timestamp":"2022-08-12T13:30:04.700554651Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"460","timestamp":"2022-08-12T13:30:04.700567196Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"461","timestamp":"2022-08-12T13:30:04.70059108Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"462","timestamp":"2022-08-12T13:30:04.70067286Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"463","timestamp":"2022-08-12T13:30:04.700674334Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"464","timestamp":"2022-08-12T13:30:04.700685415Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"465","timestamp":"2022-08-12T13:30:04.700710005Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"466","timestamp":"2022-08-12T13:30:04.700767726Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"467","timestamp":"2022-08-12T13:30:04.70076945Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"468","timestamp":"2022-08-12T13:30:04.700804243Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"469","timestamp":"2022-08-12T13:30:04.700827468Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"470","timestamp":"2022-08-12T13:30:04.700894527Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"471","timestamp":"2022-08-12T13:30:04.700896113Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"472","timestamp":"2022-08-12T13:30:04.700907008Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"473","timestamp":"2022-08-12T13:30:04.700931202Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"474","timestamp":"2022-08-12T13:30:04.7010021Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"475","timestamp":"2022-08-12T13:30:04.701003644Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"476","timestamp":"2022-08-12T13:30:04.701014805Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"477","timestamp":"2022-08-12T13:30:04.701039215Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"478","timestamp":"2022-08-12T13:30:04.70110792Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"479","timestamp":"2022-08-12T13:30:04.701109477Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"480","timestamp":"2022-08-12T13:30:04.701121995Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"481","timestamp":"2022-08-12T13:30:04.701146371Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"482","timestamp":"2022-08-12T13:30:04.701201514Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"483","timestamp":"2022-08-12T13:30:04.701202942Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"484","timestamp":"2022-08-12T13:30:04.701213855Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"485","timestamp":"2022-08-12T13:30:04.701238883Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"486","timestamp":"2022-08-12T13:30:04.701298431Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"487","timestamp":"2022-08-12T13:30:04.701299928Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"488","timestamp":"2022-08-12T13:30:04.701310874Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"489","timestamp":"2022-08-12T13:30:04.701335722Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"490","timestamp":"2022-08-12T13:30:04.701381974Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"491","timestamp":"2022-08-12T13:30:04.70138328Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"492","timestamp":"2022-08-12T13:30:04.701395934Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"493","timestamp":"2022-08-12T13:30:04.701419614Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"494","timestamp":"2022-08-12T13:30:04.701467277Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"495","timestamp":"2022-08-12T13:30:04.701468843Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"496","timestamp":"2022-08-12T13:30:04.701484172Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"497","timestamp":"2022-08-12T13:30:04.70150655Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"498","timestamp":"2022-08-12T13:30:04.701561836Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"499","timestamp":"2022-08-12T13:30:04.701563363Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"500","timestamp":"2022-08-12T13:30:04.70157438Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"501","timestamp":"2022-08-12T13:30:04.70159676Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"502","timestamp":"2022-08-12T13:30:04.701645134Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"503","timestamp":"2022-08-12T13:30:04.701646504Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"504","timestamp":"2022-08-12T13:30:04.701657604Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"505","timestamp":"2022-08-12T13:30:04.701679223Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"506","timestamp":"2022-08-12T13:30:04.701746374Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"507","timestamp":"2022-08-12T13:30:04.701747996Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"508","timestamp":"2022-08-12T13:30:04.701771132Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"509","timestamp":"2022-08-12T13:30:04.701793976Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"510","timestamp":"2022-08-12T13:30:04.701857158Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"511","timestamp":"2022-08-12T13:30:04.701858648Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"512","timestamp":"2022-08-12T13:30:04.701869489Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"513","timestamp":"2022-08-12T13:30:04.70189186Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"514","timestamp":"2022-08-12T13:30:04.70194361Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"515","timestamp":"2022-08-12T13:30:04.701944928Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"516","timestamp":"2022-08-12T13:30:04.701956241Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"517","timestamp":"2022-08-12T13:30:04.7019784Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"518","timestamp":"2022-08-12T13:30:04.702037463Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"519","timestamp":"2022-08-12T13:30:04.70203889Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"520","timestamp":"2022-08-12T13:30:04.702051432Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"521","timestamp":"2022-08-12T13:30:04.702074461Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"522","timestamp":"2022-08-12T13:30:04.702129706Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"523","timestamp":"2022-08-12T13:30:04.702131224Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"524","timestamp":"2022-08-12T13:30:04.702159656Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"525","timestamp":"2022-08-12T13:30:04.702161136Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"526","timestamp":"2022-08-12T13:30:04.702212905Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"527","timestamp":"2022-08-12T13:30:04.702214393Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"528","timestamp":"2022-08-12T13:30:04.702225942Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"529","timestamp":"2022-08-12T13:30:04.702250332Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"530","timestamp":"2022-08-12T13:30:04.70230389Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"531","timestamp":"2022-08-12T13:30:04.702305398Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"532","timestamp":"2022-08-12T13:30:04.702315851Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"533","timestamp":"2022-08-12T13:30:04.702338206Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"534","timestamp":"2022-08-12T13:30:04.70239301Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"535","timestamp":"2022-08-12T13:30:04.702394533Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"536","timestamp":"2022-08-12T13:30:04.702405636Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"537","timestamp":"2022-08-12T13:30:04.702427381Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"538","timestamp":"2022-08-12T13:30:04.702481535Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"539","timestamp":"2022-08-12T13:30:04.702482826Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"540","timestamp":"2022-08-12T13:30:04.702493337Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"541","timestamp":"2022-08-12T13:30:04.702515871Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"542","timestamp":"2022-08-12T13:30:04.702570853Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"543","timestamp":"2022-08-12T13:30:04.70257297Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"544","timestamp":"2022-08-12T13:30:04.702584051Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"545","timestamp":"2022-08-12T13:30:04.702607414Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"546","timestamp":"2022-08-12T13:30:04.702661289Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"547","timestamp":"2022-08-12T13:30:04.702662817Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"548","timestamp":"2022-08-12T13:30:04.702672982Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"549","timestamp":"2022-08-12T13:30:04.702696363Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"550","timestamp":"2022-08-12T13:30:04.702745796Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"551","timestamp":"2022-08-12T13:30:04.702747264Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"552","timestamp":"2022-08-12T13:30:04.70275758Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"553","timestamp":"2022-08-12T13:30:04.702780471Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"554","timestamp":"2022-08-12T13:30:04.702845903Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"555","timestamp":"2022-08-12T13:30:04.70284778Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"556","timestamp":"2022-08-12T13:30:04.70286042Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"557","timestamp":"2022-08-12T13:30:04.70290041Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"558","timestamp":"2022-08-12T13:30:04.702953273Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"559","timestamp":"2022-08-12T13:30:04.702955295Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"560","timestamp":"2022-08-12T13:30:04.702984203Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"561","timestamp":"2022-08-12T13:30:04.703046168Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"562","timestamp":"2022-08-12T13:30:04.703094441Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"563","timestamp":"2022-08-12T13:30:04.703096131Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"564","timestamp":"2022-08-12T13:30:04.703108827Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"565","timestamp":"2022-08-12T13:30:04.703151795Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"566","timestamp":"2022-08-12T13:30:04.703209725Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"567","timestamp":"2022-08-12T13:30:04.703211279Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"568","timestamp":"2022-08-12T13:30:04.703225085Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"569","timestamp":"2022-08-12T13:30:04.703257313Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"570","timestamp":"2022-08-12T13:30:04.703332939Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"571","timestamp":"2022-08-12T13:30:04.703334475Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"572","timestamp":"2022-08-12T13:30:04.703345816Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"573","timestamp":"2022-08-12T13:30:04.703371928Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"574","timestamp":"2022-08-12T13:30:04.703438519Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"575","timestamp":"2022-08-12T13:30:04.703440095Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"576","timestamp":"2022-08-12T13:30:04.703453879Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"577","timestamp":"2022-08-12T13:30:04.703482659Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"578","timestamp":"2022-08-12T13:30:04.70359583Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"579","timestamp":"2022-08-12T13:30:04.703597317Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"580","timestamp":"2022-08-12T13:30:04.703609203Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"581","timestamp":"2022-08-12T13:30:04.703640093Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"582","timestamp":"2022-08-12T13:30:04.703742086Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"583","timestamp":"2022-08-12T13:30:04.703743503Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"584","timestamp":"2022-08-12T13:30:04.70375489Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"585","timestamp":"2022-08-12T13:30:04.703779561Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"586","timestamp":"2022-08-12T13:30:04.703890052Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"587","timestamp":"2022-08-12T13:30:04.703891779Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"588","timestamp":"2022-08-12T13:30:04.703907719Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"589","timestamp":"2022-08-12T13:30:04.703953119Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"590","timestamp":"2022-08-12T13:30:04.704059216Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"591","timestamp":"2022-08-12T13:30:04.704060732Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"592","timestamp":"2022-08-12T13:30:04.704072287Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"593","timestamp":"2022-08-12T13:30:04.70410801Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"594","timestamp":"2022-08-12T13:30:04.704180498Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"595","timestamp":"2022-08-12T13:30:04.704182009Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"596","timestamp":"2022-08-12T13:30:04.704193724Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"597","timestamp":"2022-08-12T13:30:04.704217116Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"598","timestamp":"2022-08-12T13:30:04.704293989Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"599","timestamp":"2022-08-12T13:30:04.704295347Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"600","timestamp":"2022-08-12T13:30:04.704308533Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"601","timestamp":"2022-08-12T13:30:04.704332043Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"602","timestamp":"2022-08-12T13:30:04.7044051Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"603","timestamp":"2022-08-12T13:30:04.704406453Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"604","timestamp":"2022-08-12T13:30:04.704417353Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"605","timestamp":"2022-08-12T13:30:04.704440176Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"606","timestamp":"2022-08-12T13:30:04.704519933Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"607","timestamp":"2022-08-12T13:30:04.70452133Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"608","timestamp":"2022-08-12T13:30:04.704534809Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"609","timestamp":"2022-08-12T13:30:04.704558533Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"610","timestamp":"2022-08-12T13:30:04.704624173Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"611","timestamp":"2022-08-12T13:30:04.704625573Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"612","timestamp":"2022-08-12T13:30:04.704636102Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"613","timestamp":"2022-08-12T13:30:04.704657866Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"614","timestamp":"2022-08-12T13:30:04.704714659Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"615","timestamp":"2022-08-12T13:30:04.70471608Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"616","timestamp":"2022-08-12T13:30:04.704727384Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"617","timestamp":"2022-08-12T13:30:04.70475086Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"618","timestamp":"2022-08-12T13:30:04.704883009Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"619","timestamp":"2022-08-12T13:30:04.704884973Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"620","timestamp":"2022-08-12T13:30:04.704896004Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"621","timestamp":"2022-08-12T13:30:04.704920169Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"622","timestamp":"2022-08-12T13:30:04.704990135Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"623","timestamp":"2022-08-12T13:30:04.704991597Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"624","timestamp":"2022-08-12T13:30:04.705002334Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"625","timestamp":"2022-08-12T13:30:04.705023982Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"626","timestamp":"2022-08-12T13:30:04.705087983Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"627","timestamp":"2022-08-12T13:30:04.705089826Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"628","timestamp":"2022-08-12T13:30:04.705100979Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"629","timestamp":"2022-08-12T13:30:04.70512312Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"630","timestamp":"2022-08-12T13:30:04.705189122Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"631","timestamp":"2022-08-12T13:30:04.705190673Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"632","timestamp":"2022-08-12T13:30:04.705201969Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"633","timestamp":"2022-08-12T13:30:04.705224357Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"634","timestamp":"2022-08-12T13:30:04.705310603Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"635","timestamp":"2022-08-12T13:30:04.705312048Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"636","timestamp":"2022-08-12T13:30:04.705322383Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"637","timestamp":"2022-08-12T13:30:04.70534587Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"638","timestamp":"2022-08-12T13:30:04.705409513Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"639","timestamp":"2022-08-12T13:30:04.705410943Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"640","timestamp":"2022-08-12T13:30:04.705425433Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"641","timestamp":"2022-08-12T13:30:04.70544809Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"642","timestamp":"2022-08-12T13:30:04.705507883Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"643","timestamp":"2022-08-12T13:30:04.705509746Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"644","timestamp":"2022-08-12T13:30:04.705521201Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"645","timestamp":"2022-08-12T13:30:04.705542864Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"646","timestamp":"2022-08-12T13:30:04.705603098Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"647","timestamp":"2022-08-12T13:30:04.70560443Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"648","timestamp":"2022-08-12T13:30:04.705615912Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"649","timestamp":"2022-08-12T13:30:04.705637946Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"650","timestamp":"2022-08-12T13:30:04.705766838Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"651","timestamp":"2022-08-12T13:30:04.70588959Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"652","timestamp":"2022-08-12T13:30:04.705896459Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"653","timestamp":"2022-08-12T13:30:04.705910605Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"654","timestamp":"2022-08-12T13:30:04.706052465Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"655","timestamp":"2022-08-12T13:30:04.706054195Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"656","timestamp":"2022-08-12T13:30:04.706065743Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"657","timestamp":"2022-08-12T13:30:04.706114943Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"658","timestamp":"2022-08-12T13:30:04.706188344Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"659","timestamp":"2022-08-12T13:30:04.706190134Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"660","timestamp":"2022-08-12T13:30:04.706201666Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"661","timestamp":"2022-08-12T13:30:04.706234783Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"662","timestamp":"2022-08-12T13:30:04.706321398Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"663","timestamp":"2022-08-12T13:30:04.70632295Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"664","timestamp":"2022-08-12T13:30:04.706334209Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"665","timestamp":"2022-08-12T13:30:04.706366095Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"666","timestamp":"2022-08-12T13:30:04.706457669Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"667","timestamp":"2022-08-12T13:30:04.706459107Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"668","timestamp":"2022-08-12T13:30:04.706469863Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"669","timestamp":"2022-08-12T13:30:04.706494218Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"670","timestamp":"2022-08-12T13:30:04.706578986Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"671","timestamp":"2022-08-12T13:30:04.706581809Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"672","timestamp":"2022-08-12T13:30:04.70659846Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"673","timestamp":"2022-08-12T13:30:04.706624482Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"674","timestamp":"2022-08-12T13:30:04.706698446Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"675","timestamp":"2022-08-12T13:30:04.706700361Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"676","timestamp":"2022-08-12T13:30:04.706714027Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"677","timestamp":"2022-08-12T13:30:04.706743676Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"678","timestamp":"2022-08-12T13:30:04.706870599Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"679","timestamp":"2022-08-12T13:30:04.706872584Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"680","timestamp":"2022-08-12T13:30:04.7068853Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"681","timestamp":"2022-08-12T13:30:04.706933435Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"682","timestamp":"2022-08-12T13:30:04.707013541Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"683","timestamp":"2022-08-12T13:30:04.707015012Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"684","timestamp":"2022-08-12T13:30:04.707027844Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"685","timestamp":"2022-08-12T13:30:04.707056491Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"686","timestamp":"2022-08-12T13:30:04.707146544Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"687","timestamp":"2022-08-12T13:30:04.707148249Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"688","timestamp":"2022-08-12T13:30:04.707159028Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"689","timestamp":"2022-08-12T13:30:04.707186468Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"690","timestamp":"2022-08-12T13:30:04.707248597Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"691","timestamp":"2022-08-12T13:30:04.707250262Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"692","timestamp":"2022-08-12T13:30:04.707261454Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"693","timestamp":"2022-08-12T13:30:04.707284654Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"694","timestamp":"2022-08-12T13:30:04.707344038Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"695","timestamp":"2022-08-12T13:30:04.70734543Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"696","timestamp":"2022-08-12T13:30:04.707356688Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"697","timestamp":"2022-08-12T13:30:04.707380004Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"698","timestamp":"2022-08-12T13:30:04.707444475Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"699","timestamp":"2022-08-12T13:30:04.707452512Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"700","timestamp":"2022-08-12T13:30:04.707479109Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"701","timestamp":"2022-08-12T13:30:04.707513104Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"702","timestamp":"2022-08-12T13:30:04.707573415Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"703","timestamp":"2022-08-12T13:30:04.70757489Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"704","timestamp":"2022-08-12T13:30:04.707585715Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"705","timestamp":"2022-08-12T13:30:04.707609484Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"706","timestamp":"2022-08-12T13:30:04.707656123Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"707","timestamp":"2022-08-12T13:30:04.707657633Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"708","timestamp":"2022-08-12T13:30:04.707670601Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"709","timestamp":"2022-08-12T13:30:04.707693525Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"710","timestamp":"2022-08-12T13:30:04.707752155Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"711","timestamp":"2022-08-12T13:30:04.707753711Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"712","timestamp":"2022-08-12T13:30:04.707764721Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"713","timestamp":"2022-08-12T13:30:04.707788294Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"714","timestamp":"2022-08-12T13:30:04.707838814Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"715","timestamp":"2022-08-12T13:30:04.707840132Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"716","timestamp":"2022-08-12T13:30:04.707850656Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"717","timestamp":"2022-08-12T13:30:04.707872226Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"718","timestamp":"2022-08-12T13:30:04.707929428Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"719","timestamp":"2022-08-12T13:30:04.707930885Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"720","timestamp":"2022-08-12T13:30:04.70794376Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"721","timestamp":"2022-08-12T13:30:04.707966601Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"722","timestamp":"2022-08-12T13:30:04.708034262Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"723","timestamp":"2022-08-12T13:30:04.708035589Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"724","timestamp":"2022-08-12T13:30:04.708045668Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"725","timestamp":"2022-08-12T13:30:04.708068205Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"726","timestamp":"2022-08-12T13:30:04.708178754Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"727","timestamp":"2022-08-12T13:30:04.708181443Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"728","timestamp":"2022-08-12T13:30:04.708232434Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"729","timestamp":"2022-08-12T13:30:04.708234571Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"730","timestamp":"2022-08-12T13:30:04.708369392Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"731","timestamp":"2022-08-12T13:30:04.708372512Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"732","timestamp":"2022-08-12T13:30:04.708395368Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"733","timestamp":"2022-08-12T13:30:04.708445942Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"734","timestamp":"2022-08-12T13:30:04.708553658Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"735","timestamp":"2022-08-12T13:30:04.708556006Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"736","timestamp":"2022-08-12T13:30:04.708575383Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"737","timestamp":"2022-08-12T13:30:04.708613754Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"738","timestamp":"2022-08-12T13:30:04.708726841Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"739","timestamp":"2022-08-12T13:30:04.708729311Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"740","timestamp":"2022-08-12T13:30:04.708745908Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"741","timestamp":"2022-08-12T13:30:04.708777182Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"742","timestamp":"2022-08-12T13:30:04.708935272Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"743","timestamp":"2022-08-12T13:30:04.708937344Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"744","timestamp":"2022-08-12T13:30:04.708954836Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"745","timestamp":"2022-08-12T13:30:04.709000071Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"746","timestamp":"2022-08-12T13:30:04.70912482Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"747","timestamp":"2022-08-12T13:30:04.709128254Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"748","timestamp":"2022-08-12T13:30:04.709150148Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"749","timestamp":"2022-08-12T13:30:04.709210662Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"750","timestamp":"2022-08-12T13:30:04.709309652Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"751","timestamp":"2022-08-12T13:30:04.709312103Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"752","timestamp":"2022-08-12T13:30:04.709331811Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"753","timestamp":"2022-08-12T13:30:04.709382885Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"754","timestamp":"2022-08-12T13:30:04.709431308Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"755","timestamp":"2022-08-12T13:30:04.709433057Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"756","timestamp":"2022-08-12T13:30:04.709451889Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"757","timestamp":"2022-08-12T13:30:04.709491313Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"758","timestamp":"2022-08-12T13:30:04.709590523Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"759","timestamp":"2022-08-12T13:30:04.709592009Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"760","timestamp":"2022-08-12T13:30:04.709605244Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"761","timestamp":"2022-08-12T13:30:04.709636937Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"762","timestamp":"2022-08-12T13:30:04.709706196Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"763","timestamp":"2022-08-12T13:30:04.709708061Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"764","timestamp":"2022-08-12T13:30:04.709727055Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"765","timestamp":"2022-08-12T13:30:04.709759537Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"766","timestamp":"2022-08-12T13:30:04.709821448Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"767","timestamp":"2022-08-12T13:30:04.709823104Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"768","timestamp":"2022-08-12T13:30:04.709833983Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"769","timestamp":"2022-08-12T13:30:04.709863828Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"770","timestamp":"2022-08-12T13:30:04.709974918Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"771","timestamp":"2022-08-12T13:30:04.709977426Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"772","timestamp":"2022-08-12T13:30:04.709991337Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"773","timestamp":"2022-08-12T13:30:04.710025274Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"774","timestamp":"2022-08-12T13:30:04.71010215Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"775","timestamp":"2022-08-12T13:30:04.710103771Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"776","timestamp":"2022-08-12T13:30:04.710114715Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"777","timestamp":"2022-08-12T13:30:04.71014817Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"778","timestamp":"2022-08-12T13:30:04.710215554Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"779","timestamp":"2022-08-12T13:30:04.710216994Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"780","timestamp":"2022-08-12T13:30:04.710229487Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"781","timestamp":"2022-08-12T13:30:04.710253338Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"782","timestamp":"2022-08-12T13:30:04.710318654Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"783","timestamp":"2022-08-12T13:30:04.710320067Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"784","timestamp":"2022-08-12T13:30:04.710330402Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"785","timestamp":"2022-08-12T13:30:04.710354084Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"786","timestamp":"2022-08-12T13:30:04.710418256Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"787","timestamp":"2022-08-12T13:30:04.710419756Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"788","timestamp":"2022-08-12T13:30:04.710430434Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"789","timestamp":"2022-08-12T13:30:04.710454041Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"790","timestamp":"2022-08-12T13:30:04.710512365Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"791","timestamp":"2022-08-12T13:30:04.710513768Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"792","timestamp":"2022-08-12T13:30:04.710525787Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"793","timestamp":"2022-08-12T13:30:04.710549626Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"794","timestamp":"2022-08-12T13:30:04.710615114Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"795","timestamp":"2022-08-12T13:30:04.710616478Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"796","timestamp":"2022-08-12T13:30:04.710627446Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"797","timestamp":"2022-08-12T13:30:04.710652112Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"798","timestamp":"2022-08-12T13:30:04.710716468Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"799","timestamp":"2022-08-12T13:30:04.710717858Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"800","timestamp":"2022-08-12T13:30:04.710728396Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"801","timestamp":"2022-08-12T13:30:04.710752871Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"802","timestamp":"2022-08-12T13:30:04.710837134Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"803","timestamp":"2022-08-12T13:30:04.710838506Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"804","timestamp":"2022-08-12T13:30:04.710848881Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"805","timestamp":"2022-08-12T13:30:04.7108824Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"806","timestamp":"2022-08-12T13:30:04.710962594Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"807","timestamp":"2022-08-12T13:30:04.710964116Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"808","timestamp":"2022-08-12T13:30:04.710974224Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"809","timestamp":"2022-08-12T13:30:04.710997471Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"810","timestamp":"2022-08-12T13:30:04.711072504Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"811","timestamp":"2022-08-12T13:30:04.711073919Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"812","timestamp":"2022-08-12T13:30:04.71108442Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"813","timestamp":"2022-08-12T13:30:04.71110896Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"814","timestamp":"2022-08-12T13:30:04.711168002Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"815","timestamp":"2022-08-12T13:30:04.7111694Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"816","timestamp":"2022-08-12T13:30:04.711200056Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"817","timestamp":"2022-08-12T13:30:04.711201407Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"818","timestamp":"2022-08-12T13:30:04.711284343Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"819","timestamp":"2022-08-12T13:30:04.711286559Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"820","timestamp":"2022-08-12T13:30:04.711302987Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"821","timestamp":"2022-08-12T13:30:04.711333257Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"822","timestamp":"2022-08-12T13:30:04.711461684Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"823","timestamp":"2022-08-12T13:30:04.711463275Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"824","timestamp":"2022-08-12T13:30:04.711473831Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"825","timestamp":"2022-08-12T13:30:04.711510722Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"826","timestamp":"2022-08-12T13:30:04.711776112Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"827","timestamp":"2022-08-12T13:30:04.711778088Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"828","timestamp":"2022-08-12T13:30:04.711790555Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"829","timestamp":"2022-08-12T13:30:04.711850892Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"830","timestamp":"2022-08-12T13:30:04.711955921Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"831","timestamp":"2022-08-12T13:30:04.711959057Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"832","timestamp":"2022-08-12T13:30:04.711979285Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"833","timestamp":"2022-08-12T13:30:04.712012517Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"834","timestamp":"2022-08-12T13:30:04.712141797Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"835","timestamp":"2022-08-12T13:30:04.712144941Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"836","timestamp":"2022-08-12T13:30:04.712167389Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"837","timestamp":"2022-08-12T13:30:04.71222609Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"838","timestamp":"2022-08-12T13:30:04.712275903Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"839","timestamp":"2022-08-12T13:30:04.71227873Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"840","timestamp":"2022-08-12T13:30:04.712298541Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"841","timestamp":"2022-08-12T13:30:04.712346086Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"844","timestamp":"2022-08-12T13:30:04.712432623Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"842","timestamp":"2022-08-12T13:30:04.712417798Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"845","timestamp":"2022-08-12T13:30:04.712475176Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"843","timestamp":"2022-08-12T13:30:04.712419615Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"846","timestamp":"2022-08-12T13:30:04.712541331Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"847","timestamp":"2022-08-12T13:30:04.712544343Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"848","timestamp":"2022-08-12T13:30:04.712565343Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"849","timestamp":"2022-08-12T13:30:04.71260141Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"850","timestamp":"2022-08-12T13:30:04.712679533Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"851","timestamp":"2022-08-12T13:30:04.712682047Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"852","timestamp":"2022-08-12T13:30:04.71269993Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"853","timestamp":"2022-08-12T13:30:04.712744578Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"854","timestamp":"2022-08-12T13:30:04.712832526Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"855","timestamp":"2022-08-12T13:30:04.712835083Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"856","timestamp":"2022-08-12T13:30:04.712855748Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"857","timestamp":"2022-08-12T13:30:04.712901117Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"858","timestamp":"2022-08-12T13:30:04.712948278Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"859","timestamp":"2022-08-12T13:30:04.712950652Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"860","timestamp":"2022-08-12T13:30:04.712972108Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"861","timestamp":"2022-08-12T13:30:04.713013007Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"862","timestamp":"2022-08-12T13:30:04.713086755Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"863","timestamp":"2022-08-12T13:30:04.713095643Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"864","timestamp":"2022-08-12T13:30:04.713124847Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"865","timestamp":"2022-08-12T13:30:04.713193224Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"868","timestamp":"2022-08-12T13:30:04.713242747Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"866","timestamp":"2022-08-12T13:30:04.713230415Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"869","timestamp":"2022-08-12T13:30:04.713273546Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"867","timestamp":"2022-08-12T13:30:04.713231837Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"870","timestamp":"2022-08-12T13:30:04.713374227Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"871","timestamp":"2022-08-12T13:30:04.713376026Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"876","timestamp":"2022-08-12T13:30:04.713494103Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"877","timestamp":"2022-08-12T13:30:04.713560486Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"878","timestamp":"2022-08-12T13:30:04.713617852Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"872","timestamp":"2022-08-12T13:30:04.71338764Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"879","timestamp":"2022-08-12T13:30:04.713619264Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"873","timestamp":"2022-08-12T13:30:04.71343859Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"874","timestamp":"2022-08-12T13:30:04.713480681Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"875","timestamp":"2022-08-12T13:30:04.713482158Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"880","timestamp":"2022-08-12T13:30:04.713635908Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"881","timestamp":"2022-08-12T13:30:04.713666749Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"882","timestamp":"2022-08-12T13:30:04.713737208Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"883","timestamp":"2022-08-12T13:30:04.713738675Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"884","timestamp":"2022-08-12T13:30:04.713749906Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"885","timestamp":"2022-08-12T13:30:04.713776348Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"886","timestamp":"2022-08-12T13:30:04.713842556Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"887","timestamp":"2022-08-12T13:30:04.713844333Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"888","timestamp":"2022-08-12T13:30:04.713856017Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"889","timestamp":"2022-08-12T13:30:04.713918858Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"890","timestamp":"2022-08-12T13:30:04.713957075Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"891","timestamp":"2022-08-12T13:30:04.713958725Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"892","timestamp":"2022-08-12T13:30:04.71397766Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"893","timestamp":"2022-08-12T13:30:04.713997861Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"894","timestamp":"2022-08-12T13:30:04.714068363Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"895","timestamp":"2022-08-12T13:30:04.714069854Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"904","timestamp":"2022-08-12T13:30:04.71441155Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"905","timestamp":"2022-08-12T13:30:04.714443146Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"906","timestamp":"2022-08-12T13:30:04.714526487Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"907","timestamp":"2022-08-12T13:30:04.71452801Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"900","timestamp":"2022-08-12T13:30:04.714259517Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"901","timestamp":"2022-08-12T13:30:04.71431771Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"896","timestamp":"2022-08-12T13:30:04.714081946Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"902","timestamp":"2022-08-12T13:30:04.714398763Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"897","timestamp":"2022-08-12T13:30:04.714131856Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"903","timestamp":"2022-08-12T13:30:04.714400278Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"898","timestamp":"2022-08-12T13:30:04.714221145Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"899","timestamp":"2022-08-12T13:30:04.714222816Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"908","timestamp":"2022-08-12T13:30:04.714539876Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"909","timestamp":"2022-08-12T13:30:04.714579616Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"910","timestamp":"2022-08-12T13:30:04.714648781Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"911","timestamp":"2022-08-12T13:30:04.714650352Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"912","timestamp":"2022-08-12T13:30:04.714661169Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"913","timestamp":"2022-08-12T13:30:04.714687341Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"914","timestamp":"2022-08-12T13:30:04.714752976Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"915","timestamp":"2022-08-12T13:30:04.71475437Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"916","timestamp":"2022-08-12T13:30:04.714766934Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"917","timestamp":"2022-08-12T13:30:04.714791959Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"918","timestamp":"2022-08-12T13:30:04.714855719Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"919","timestamp":"2022-08-12T13:30:04.714857136Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"924","timestamp":"2022-08-12T13:30:04.714966515Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"920","timestamp":"2022-08-12T13:30:04.71486945Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"925","timestamp":"2022-08-12T13:30:04.714989623Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"921","timestamp":"2022-08-12T13:30:04.714897783Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"926","timestamp":"2022-08-12T13:30:04.715051119Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"922","timestamp":"2022-08-12T13:30:04.714953405Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"927","timestamp":"2022-08-12T13:30:04.715052519Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"923","timestamp":"2022-08-12T13:30:04.714954731Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"928","timestamp":"2022-08-12T13:30:04.715063486Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"929","timestamp":"2022-08-12T13:30:04.715086245Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"930","timestamp":"2022-08-12T13:30:04.71514682Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"931","timestamp":"2022-08-12T13:30:04.715208913Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"932","timestamp":"2022-08-12T13:30:04.715309406Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"933","timestamp":"2022-08-12T13:30:04.715319793Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"934","timestamp":"2022-08-12T13:30:04.715330232Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"935","timestamp":"2022-08-12T13:30:04.715355303Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"936","timestamp":"2022-08-12T13:30:04.715367929Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"937","timestamp":"2022-08-12T13:30:04.715388934Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"938","timestamp":"2022-08-12T13:30:04.715536443Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"939","timestamp":"2022-08-12T13:30:04.715537919Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"940","timestamp":"2022-08-12T13:30:04.715548611Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"941","timestamp":"2022-08-12T13:30:04.715578812Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"942","timestamp":"2022-08-12T13:30:04.715644144Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"943","timestamp":"2022-08-12T13:30:04.71564598Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"944","timestamp":"2022-08-12T13:30:04.715657848Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"945","timestamp":"2022-08-12T13:30:04.715697271Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"946","timestamp":"2022-08-12T13:30:04.715762211Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"947","timestamp":"2022-08-12T13:30:04.715764594Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"952","timestamp":"2022-08-12T13:30:04.715951386Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"953","timestamp":"2022-08-12T13:30:04.715985421Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"954","timestamp":"2022-08-12T13:30:04.71606346Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"955","timestamp":"2022-08-12T13:30:04.716065057Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"948","timestamp":"2022-08-12T13:30:04.715784626Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"949","timestamp":"2022-08-12T13:30:04.715819265Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"950","timestamp":"2022-08-12T13:30:04.715926101Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"951","timestamp":"2022-08-12T13:30:04.715928999Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"956","timestamp":"2022-08-12T13:30:04.716078469Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"957","timestamp":"2022-08-12T13:30:04.716105203Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"958","timestamp":"2022-08-12T13:30:04.716186805Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"959","timestamp":"2022-08-12T13:30:04.716189249Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"960","timestamp":"2022-08-12T13:30:04.716209061Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"961","timestamp":"2022-08-12T13:30:04.716261576Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"962","timestamp":"2022-08-12T13:30:04.716300495Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"963","timestamp":"2022-08-12T13:30:04.716301971Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"964","timestamp":"2022-08-12T13:30:04.716312989Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"965","timestamp":"2022-08-12T13:30:04.716348322Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"966","timestamp":"2022-08-12T13:30:04.716423956Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"967","timestamp":"2022-08-12T13:30:04.716426315Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"968","timestamp":"2022-08-12T13:30:04.716447423Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"969","timestamp":"2022-08-12T13:30:04.716509994Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"970","timestamp":"2022-08-12T13:30:04.71658427Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"971","timestamp":"2022-08-12T13:30:04.716585802Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"972","timestamp":"2022-08-12T13:30:04.716597559Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"973","timestamp":"2022-08-12T13:30:04.716640621Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"974","timestamp":"2022-08-12T13:30:04.716699467Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"975","timestamp":"2022-08-12T13:30:04.716702214Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"976","timestamp":"2022-08-12T13:30:04.716731172Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"977","timestamp":"2022-08-12T13:30:04.716795598Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"978","timestamp":"2022-08-12T13:30:04.716888316Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"979","timestamp":"2022-08-12T13:30:04.716891115Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"980","timestamp":"2022-08-12T13:30:04.716912043Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"981","timestamp":"2022-08-12T13:30:04.716949806Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"982","timestamp":"2022-08-12T13:30:04.717044262Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"983","timestamp":"2022-08-12T13:30:04.717046304Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"984","timestamp":"2022-08-12T13:30:04.717058535Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"985","timestamp":"2022-08-12T13:30:04.717089878Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"988","timestamp":"2022-08-12T13:30:04.717224621Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"986","timestamp":"2022-08-12T13:30:04.717201882Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"989","timestamp":"2022-08-12T13:30:04.717255035Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"987","timestamp":"2022-08-12T13:30:04.71720484Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"990","timestamp":"2022-08-12T13:30:04.717361332Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"991","timestamp":"2022-08-12T13:30:04.717363063Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"992","timestamp":"2022-08-12T13:30:04.717374202Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"993","timestamp":"2022-08-12T13:30:04.717422883Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"994","timestamp":"2022-08-12T13:30:04.717540795Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"995","timestamp":"2022-08-12T13:30:04.717542899Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"996","timestamp":"2022-08-12T13:30:04.717559179Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1000","timestamp":"2022-08-12T13:30:04.71770325Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"997","timestamp":"2022-08-12T13:30:04.717602673Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1001","timestamp":"2022-08-12T13:30:04.717748364Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"998","timestamp":"2022-08-12T13:30:04.717689994Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1002","timestamp":"2022-08-12T13:30:04.717810876Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"999","timestamp":"2022-08-12T13:30:04.717691639Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1003","timestamp":"2022-08-12T13:30:04.717813824Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1004","timestamp":"2022-08-12T13:30:04.717825579Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1005","timestamp":"2022-08-12T13:30:04.717850867Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1006","timestamp":"2022-08-12T13:30:04.717913556Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1007","timestamp":"2022-08-12T13:30:04.717915015Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1008","timestamp":"2022-08-12T13:30:04.717925497Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1009","timestamp":"2022-08-12T13:30:04.717947864Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1010","timestamp":"2022-08-12T13:30:04.718040632Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1011","timestamp":"2022-08-12T13:30:04.718042702Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1012","timestamp":"2022-08-12T13:30:04.718098444Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1013","timestamp":"2022-08-12T13:30:04.718151358Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1014","timestamp":"2022-08-12T13:30:04.718234563Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1015","timestamp":"2022-08-12T13:30:04.718236201Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1016","timestamp":"2022-08-12T13:30:04.718251774Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1017","timestamp":"2022-08-12T13:30:04.718279106Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1018","timestamp":"2022-08-12T13:30:04.71836407Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1019","timestamp":"2022-08-12T13:30:04.71836553Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1020","timestamp":"2022-08-12T13:30:04.718378678Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1021","timestamp":"2022-08-12T13:30:04.718403592Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1022","timestamp":"2022-08-12T13:30:04.718492334Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1023","timestamp":"2022-08-12T13:30:04.718494063Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1024","timestamp":"2022-08-12T13:30:04.718505552Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1025","timestamp":"2022-08-12T13:30:04.718558491Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1026","timestamp":"2022-08-12T13:30:04.718634674Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1027","timestamp":"2022-08-12T13:30:04.71863619Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1028","timestamp":"2022-08-12T13:30:04.718647076Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1029","timestamp":"2022-08-12T13:30:04.718672497Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1030","timestamp":"2022-08-12T13:30:04.718735921Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1031","timestamp":"2022-08-12T13:30:04.718737498Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1032","timestamp":"2022-08-12T13:30:04.718757782Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1033","timestamp":"2022-08-12T13:30:04.71878533Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1034","timestamp":"2022-08-12T13:30:04.718850074Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1035","timestamp":"2022-08-12T13:30:04.718851545Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1036","timestamp":"2022-08-12T13:30:04.718862618Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1037","timestamp":"2022-08-12T13:30:04.718888289Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1038","timestamp":"2022-08-12T13:30:04.71899266Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1039","timestamp":"2022-08-12T13:30:04.718994119Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1040","timestamp":"2022-08-12T13:30:04.719006465Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1041","timestamp":"2022-08-12T13:30:04.719034067Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1042","timestamp":"2022-08-12T13:30:04.719117434Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1043","timestamp":"2022-08-12T13:30:04.719118934Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1044","timestamp":"2022-08-12T13:30:04.719129992Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1045","timestamp":"2022-08-12T13:30:04.719154181Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1046","timestamp":"2022-08-12T13:30:04.719220595Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1047","timestamp":"2022-08-12T13:30:04.719222004Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1048","timestamp":"2022-08-12T13:30:04.71923287Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1049","timestamp":"2022-08-12T13:30:04.719257141Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1050","timestamp":"2022-08-12T13:30:04.719305938Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1051","timestamp":"2022-08-12T13:30:04.719307428Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1052","timestamp":"2022-08-12T13:30:04.719320222Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1053","timestamp":"2022-08-12T13:30:04.719344568Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1054","timestamp":"2022-08-12T13:30:04.719394472Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1055","timestamp":"2022-08-12T13:30:04.719395892Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1056","timestamp":"2022-08-12T13:30:04.719406265Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1057","timestamp":"2022-08-12T13:30:04.719435252Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1058","timestamp":"2022-08-12T13:30:04.719482663Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1059","timestamp":"2022-08-12T13:30:04.719484978Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1060","timestamp":"2022-08-12T13:30:04.719500193Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1061","timestamp":"2022-08-12T13:30:04.719544044Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1062","timestamp":"2022-08-12T13:30:04.719601622Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1063","timestamp":"2022-08-12T13:30:04.719603628Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1064","timestamp":"2022-08-12T13:30:04.719620626Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1065","timestamp":"2022-08-12T13:30:04.71966462Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1066","timestamp":"2022-08-12T13:30:04.719718699Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1067","timestamp":"2022-08-12T13:30:04.719720137Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1068","timestamp":"2022-08-12T13:30:04.719730749Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1069","timestamp":"2022-08-12T13:30:04.719755893Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1070","timestamp":"2022-08-12T13:30:04.719820477Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1071","timestamp":"2022-08-12T13:30:04.719823913Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1072","timestamp":"2022-08-12T13:30:04.719851384Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1073","timestamp":"2022-08-12T13:30:04.719900268Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1074","timestamp":"2022-08-12T13:30:04.719989798Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1075","timestamp":"2022-08-12T13:30:04.719991293Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1076","timestamp":"2022-08-12T13:30:04.720002148Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1080","timestamp":"2022-08-12T13:30:04.720109938Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1077","timestamp":"2022-08-12T13:30:04.720028646Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1081","timestamp":"2022-08-12T13:30:04.720135214Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1078","timestamp":"2022-08-12T13:30:04.72009678Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1082","timestamp":"2022-08-12T13:30:04.720233666Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1079","timestamp":"2022-08-12T13:30:04.720098245Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1083","timestamp":"2022-08-12T13:30:04.720235717Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1084","timestamp":"2022-08-12T13:30:04.720250266Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1085","timestamp":"2022-08-12T13:30:04.72029262Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1086","timestamp":"2022-08-12T13:30:04.720355963Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1087","timestamp":"2022-08-12T13:30:04.720357702Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1092","timestamp":"2022-08-12T13:30:04.720491846Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1093","timestamp":"2022-08-12T13:30:04.720522913Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1088","timestamp":"2022-08-12T13:30:04.720372571Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1094","timestamp":"2022-08-12T13:30:04.720582715Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1095","timestamp":"2022-08-12T13:30:04.720584485Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1089","timestamp":"2022-08-12T13:30:04.720410812Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1090","timestamp":"2022-08-12T13:30:04.720478491Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1091","timestamp":"2022-08-12T13:30:04.720480218Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1096","timestamp":"2022-08-12T13:30:04.720598203Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1097","timestamp":"2022-08-12T13:30:04.720626808Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1098","timestamp":"2022-08-12T13:30:04.72067591Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1099","timestamp":"2022-08-12T13:30:04.720677401Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1100","timestamp":"2022-08-12T13:30:04.720690184Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1101","timestamp":"2022-08-12T13:30:04.720715219Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1102","timestamp":"2022-08-12T13:30:04.720769041Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1103","timestamp":"2022-08-12T13:30:04.720770757Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1104","timestamp":"2022-08-12T13:30:04.720805757Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1105","timestamp":"2022-08-12T13:30:04.72083091Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1106","timestamp":"2022-08-12T13:30:04.720910654Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1107","timestamp":"2022-08-12T13:30:04.720912234Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1108","timestamp":"2022-08-12T13:30:04.720922603Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1109","timestamp":"2022-08-12T13:30:04.720961357Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1110","timestamp":"2022-08-12T13:30:04.721056935Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1112","timestamp":"2022-08-12T13:30:04.721082107Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1111","timestamp":"2022-08-12T13:30:04.721060116Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1113","timestamp":"2022-08-12T13:30:04.721109374Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1114","timestamp":"2022-08-12T13:30:04.721185212Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1115","timestamp":"2022-08-12T13:30:04.721186698Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1116","timestamp":"2022-08-12T13:30:04.721197567Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1117","timestamp":"2022-08-12T13:30:04.721230527Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1118","timestamp":"2022-08-12T13:30:04.72128482Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1119","timestamp":"2022-08-12T13:30:04.721286311Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1120","timestamp":"2022-08-12T13:30:04.721296783Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1121","timestamp":"2022-08-12T13:30:04.721330626Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1122","timestamp":"2022-08-12T13:30:04.721374349Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1123","timestamp":"2022-08-12T13:30:04.721375723Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1124","timestamp":"2022-08-12T13:30:04.721386016Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1125","timestamp":"2022-08-12T13:30:04.721417245Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1126","timestamp":"2022-08-12T13:30:04.721465258Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1127","timestamp":"2022-08-12T13:30:04.721466618Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1128","timestamp":"2022-08-12T13:30:04.721476998Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1129","timestamp":"2022-08-12T13:30:04.721507429Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1130","timestamp":"2022-08-12T13:30:04.721567582Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1131","timestamp":"2022-08-12T13:30:04.721571813Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1132","timestamp":"2022-08-12T13:30:04.721595411Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1133","timestamp":"2022-08-12T13:30:04.721642744Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1134","timestamp":"2022-08-12T13:30:04.72168199Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1135","timestamp":"2022-08-12T13:30:04.721683526Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1136","timestamp":"2022-08-12T13:30:04.721699658Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1137","timestamp":"2022-08-12T13:30:04.721757707Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1138","timestamp":"2022-08-12T13:30:04.7218142Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1139","timestamp":"2022-08-12T13:30:04.721815936Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1140","timestamp":"2022-08-12T13:30:04.721827629Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1141","timestamp":"2022-08-12T13:30:04.721857375Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1142","timestamp":"2022-08-12T13:30:04.72193039Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1143","timestamp":"2022-08-12T13:30:04.721932146Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1144","timestamp":"2022-08-12T13:30:04.721943453Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1148","timestamp":"2022-08-12T13:30:04.722041659Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1145","timestamp":"2022-08-12T13:30:04.721970231Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1149","timestamp":"2022-08-12T13:30:04.722070899Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1146","timestamp":"2022-08-12T13:30:04.722029146Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1150","timestamp":"2022-08-12T13:30:04.722121361Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1147","timestamp":"2022-08-12T13:30:04.72203073Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1151","timestamp":"2022-08-12T13:30:04.722123262Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1152","timestamp":"2022-08-12T13:30:04.722137804Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1153","timestamp":"2022-08-12T13:30:04.722165053Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1154","timestamp":"2022-08-12T13:30:04.722202637Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1155","timestamp":"2022-08-12T13:30:04.722204357Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1156","timestamp":"2022-08-12T13:30:04.722218697Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1157","timestamp":"2022-08-12T13:30:04.722248043Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1158","timestamp":"2022-08-12T13:30:04.722323697Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1159","timestamp":"2022-08-12T13:30:04.722326559Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1160","timestamp":"2022-08-12T13:30:04.722346125Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1161","timestamp":"2022-08-12T13:30:04.72239747Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1162","timestamp":"2022-08-12T13:30:04.722428438Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1163","timestamp":"2022-08-12T13:30:04.722430372Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1164","timestamp":"2022-08-12T13:30:04.722444225Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1165","timestamp":"2022-08-12T13:30:04.722477559Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1166","timestamp":"2022-08-12T13:30:04.722545503Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1167","timestamp":"2022-08-12T13:30:04.722548106Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1168","timestamp":"2022-08-12T13:30:04.722567527Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1169","timestamp":"2022-08-12T13:30:04.722603043Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1170","timestamp":"2022-08-12T13:30:04.722658737Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1171","timestamp":"2022-08-12T13:30:04.72266028Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1172","timestamp":"2022-08-12T13:30:04.722671179Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1173","timestamp":"2022-08-12T13:30:04.72270491Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1174","timestamp":"2022-08-12T13:30:04.722761636Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1175","timestamp":"2022-08-12T13:30:04.722841361Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1176","timestamp":"2022-08-12T13:30:04.722842974Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1177","timestamp":"2022-08-12T13:30:04.722854654Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1178","timestamp":"2022-08-12T13:30:04.722880331Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1181","timestamp":"2022-08-12T13:30:04.722937624Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1179","timestamp":"2022-08-12T13:30:04.722925764Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1182","timestamp":"2022-08-12T13:30:04.722961927Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1180","timestamp":"2022-08-12T13:30:04.722927287Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1183","timestamp":"2022-08-12T13:30:04.723009558Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1184","timestamp":"2022-08-12T13:30:04.723010906Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1185","timestamp":"2022-08-12T13:30:04.72302468Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1186","timestamp":"2022-08-12T13:30:04.7230504Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1187","timestamp":"2022-08-12T13:30:04.72309843Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1188","timestamp":"2022-08-12T13:30:04.723099875Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1189","timestamp":"2022-08-12T13:30:04.723110268Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1190","timestamp":"2022-08-12T13:30:04.723134788Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1191","timestamp":"2022-08-12T13:30:04.72318872Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1192","timestamp":"2022-08-12T13:30:04.723190113Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1193","timestamp":"2022-08-12T13:30:04.72320054Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1194","timestamp":"2022-08-12T13:30:04.72322426Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1195","timestamp":"2022-08-12T13:30:04.72327904Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1196","timestamp":"2022-08-12T13:30:04.723280423Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1197","timestamp":"2022-08-12T13:30:04.723290803Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1198","timestamp":"2022-08-12T13:30:04.723314474Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1199","timestamp":"2022-08-12T13:30:04.723369582Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1200","timestamp":"2022-08-12T13:30:04.723370982Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1201","timestamp":"2022-08-12T13:30:04.723381446Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1202","timestamp":"2022-08-12T13:30:04.723405137Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1203","timestamp":"2022-08-12T13:30:04.723461813Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1204","timestamp":"2022-08-12T13:30:04.723463173Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1205","timestamp":"2022-08-12T13:30:04.723473901Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1206","timestamp":"2022-08-12T13:30:04.723497556Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1207","timestamp":"2022-08-12T13:30:04.723552536Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1208","timestamp":"2022-08-12T13:30:04.723554122Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1209","timestamp":"2022-08-12T13:30:04.723565649Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1210","timestamp":"2022-08-12T13:30:04.723589159Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1211","timestamp":"2022-08-12T13:30:04.723642396Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1212","timestamp":"2022-08-12T13:30:04.723643789Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1213","timestamp":"2022-08-12T13:30:04.723707718Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1214","timestamp":"2022-08-12T13:30:04.723784035Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1215","timestamp":"2022-08-12T13:30:04.723812348Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1216","timestamp":"2022-08-12T13:30:04.72381381Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1217","timestamp":"2022-08-12T13:30:04.723825329Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1218","timestamp":"2022-08-12T13:30:04.72385155Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1219","timestamp":"2022-08-12T13:30:04.723901202Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1220","timestamp":"2022-08-12T13:30:04.723902573Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1221","timestamp":"2022-08-12T13:30:04.723918403Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1222","timestamp":"2022-08-12T13:30:04.723941755Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1223","timestamp":"2022-08-12T13:30:04.723991497Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1224","timestamp":"2022-08-12T13:30:04.723993912Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1225","timestamp":"2022-08-12T13:30:04.724008957Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1226","timestamp":"2022-08-12T13:30:04.724053334Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1227","timestamp":"2022-08-12T13:30:04.724109193Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1228","timestamp":"2022-08-12T13:30:04.724110657Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1229","timestamp":"2022-08-12T13:30:04.72412113Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1230","timestamp":"2022-08-12T13:30:04.724146191Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1231","timestamp":"2022-08-12T13:30:04.724213719Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1232","timestamp":"2022-08-12T13:30:04.724215244Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1233","timestamp":"2022-08-12T13:30:04.724226742Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1234","timestamp":"2022-08-12T13:30:04.724254342Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1235","timestamp":"2022-08-12T13:30:04.724348093Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1236","timestamp":"2022-08-12T13:30:04.724349915Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1237","timestamp":"2022-08-12T13:30:04.724361177Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1238","timestamp":"2022-08-12T13:30:04.724397888Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1239","timestamp":"2022-08-12T13:30:04.72445394Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1240","timestamp":"2022-08-12T13:30:04.724455777Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1241","timestamp":"2022-08-12T13:30:04.724466161Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1242","timestamp":"2022-08-12T13:30:04.724517422Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1243","timestamp":"2022-08-12T13:30:04.724553559Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1244","timestamp":"2022-08-12T13:30:04.724556403Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1245","timestamp":"2022-08-12T13:30:04.724567528Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1246","timestamp":"2022-08-12T13:30:04.724598314Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1247","timestamp":"2022-08-12T13:30:04.724649254Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1248","timestamp":"2022-08-12T13:30:04.724650763Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1249","timestamp":"2022-08-12T13:30:04.724663703Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1250","timestamp":"2022-08-12T13:30:04.724694939Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1251","timestamp":"2022-08-12T13:30:04.724744175Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1252","timestamp":"2022-08-12T13:30:04.724745563Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1253","timestamp":"2022-08-12T13:30:04.724756505Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1254","timestamp":"2022-08-12T13:30:04.724808177Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1255","timestamp":"2022-08-12T13:30:04.724851439Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1256","timestamp":"2022-08-12T13:30:04.724852945Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1257","timestamp":"2022-08-12T13:30:04.724863212Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1258","timestamp":"2022-08-12T13:30:04.72489549Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1259","timestamp":"2022-08-12T13:30:04.72494261Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1260","timestamp":"2022-08-12T13:30:04.724944021Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1261","timestamp":"2022-08-12T13:30:04.724954912Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1262","timestamp":"2022-08-12T13:30:04.724985556Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1263","timestamp":"2022-08-12T13:30:04.725040078Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1264","timestamp":"2022-08-12T13:30:04.725041609Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1265","timestamp":"2022-08-12T13:30:04.725052849Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1266","timestamp":"2022-08-12T13:30:04.725082432Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1267","timestamp":"2022-08-12T13:30:04.725138728Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1268","timestamp":"2022-08-12T13:30:04.725140187Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1269","timestamp":"2022-08-12T13:30:04.725151372Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1270","timestamp":"2022-08-12T13:30:04.725175776Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1271","timestamp":"2022-08-12T13:30:04.725225587Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1272","timestamp":"2022-08-12T13:30:04.725226994Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1273","timestamp":"2022-08-12T13:30:04.725237458Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1274","timestamp":"2022-08-12T13:30:04.725261546Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1275","timestamp":"2022-08-12T13:30:04.725323126Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1276","timestamp":"2022-08-12T13:30:04.725324729Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1277","timestamp":"2022-08-12T13:30:04.725337708Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1278","timestamp":"2022-08-12T13:30:04.725361718Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1279","timestamp":"2022-08-12T13:30:04.725421275Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1280","timestamp":"2022-08-12T13:30:04.725422632Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1281","timestamp":"2022-08-12T13:30:04.725432696Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1282","timestamp":"2022-08-12T13:30:04.725457678Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1283","timestamp":"2022-08-12T13:30:04.72551114Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1284","timestamp":"2022-08-12T13:30:04.725512574Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1285","timestamp":"2022-08-12T13:30:04.725523899Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1286","timestamp":"2022-08-12T13:30:04.725547565Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1287","timestamp":"2022-08-12T13:30:04.725594163Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1288","timestamp":"2022-08-12T13:30:04.725595469Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1289","timestamp":"2022-08-12T13:30:04.725605873Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1290","timestamp":"2022-08-12T13:30:04.725632579Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1291","timestamp":"2022-08-12T13:30:04.725677381Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1292","timestamp":"2022-08-12T13:30:04.725678797Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1293","timestamp":"2022-08-12T13:30:04.725689297Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1294","timestamp":"2022-08-12T13:30:04.725713233Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1295","timestamp":"2022-08-12T13:30:04.725772885Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1296","timestamp":"2022-08-12T13:30:04.725774249Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1297","timestamp":"2022-08-12T13:30:04.725784116Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1298","timestamp":"2022-08-12T13:30:04.725808942Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1299","timestamp":"2022-08-12T13:30:04.725856476Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1300","timestamp":"2022-08-12T13:30:04.725857842Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1301","timestamp":"2022-08-12T13:30:04.72586863Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1302","timestamp":"2022-08-12T13:30:04.725892592Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1303","timestamp":"2022-08-12T13:30:04.725949233Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1304","timestamp":"2022-08-12T13:30:04.7259519Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1305","timestamp":"2022-08-12T13:30:04.72597082Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1306","timestamp":"2022-08-12T13:30:04.726001502Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1307","timestamp":"2022-08-12T13:30:04.726068619Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1308","timestamp":"2022-08-12T13:30:04.726070651Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1309","timestamp":"2022-08-12T13:30:04.7260833Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1310","timestamp":"2022-08-12T13:30:04.726108317Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1311","timestamp":"2022-08-12T13:30:04.726159542Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1312","timestamp":"2022-08-12T13:30:04.726160946Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1313","timestamp":"2022-08-12T13:30:04.726174374Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1314","timestamp":"2022-08-12T13:30:04.726198185Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1315","timestamp":"2022-08-12T13:30:04.726243046Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1316","timestamp":"2022-08-12T13:30:04.726244437Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1317","timestamp":"2022-08-12T13:30:04.72625513Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1318","timestamp":"2022-08-12T13:30:04.726279139Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1319","timestamp":"2022-08-12T13:30:04.726328104Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1320","timestamp":"2022-08-12T13:30:04.726329551Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1321","timestamp":"2022-08-12T13:30:04.726340628Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1322","timestamp":"2022-08-12T13:30:04.726363983Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1325","timestamp":"2022-08-12T13:30:04.726426695Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1323","timestamp":"2022-08-12T13:30:04.726414779Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1326","timestamp":"2022-08-12T13:30:04.726450041Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1324","timestamp":"2022-08-12T13:30:04.726416144Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1327","timestamp":"2022-08-12T13:30:04.726502988Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1328","timestamp":"2022-08-12T13:30:04.726504422Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1329","timestamp":"2022-08-12T13:30:04.726515286Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1330","timestamp":"2022-08-12T13:30:04.726538294Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1331","timestamp":"2022-08-12T13:30:04.726589573Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1332","timestamp":"2022-08-12T13:30:04.726590938Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1333","timestamp":"2022-08-12T13:30:04.726604194Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1334","timestamp":"2022-08-12T13:30:04.726629202Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1335","timestamp":"2022-08-12T13:30:04.726677631Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1337","timestamp":"2022-08-12T13:30:04.726691686Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1336","timestamp":"2022-08-12T13:30:04.72667899Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1338","timestamp":"2022-08-12T13:30:04.7267156Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1339","timestamp":"2022-08-12T13:30:04.726766152Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1340","timestamp":"2022-08-12T13:30:04.726767545Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1341","timestamp":"2022-08-12T13:30:04.726779868Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1342","timestamp":"2022-08-12T13:30:04.726803769Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1343","timestamp":"2022-08-12T13:30:04.726866848Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1344","timestamp":"2022-08-12T13:30:04.726868706Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1345","timestamp":"2022-08-12T13:30:04.72688061Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1346","timestamp":"2022-08-12T13:30:04.726914339Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1347","timestamp":"2022-08-12T13:30:04.726970228Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1348","timestamp":"2022-08-12T13:30:04.726971719Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1349","timestamp":"2022-08-12T13:30:04.726986218Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1357","timestamp":"2022-08-12T13:30:04.727151561Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1350","timestamp":"2022-08-12T13:30:04.727011351Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1358","timestamp":"2022-08-12T13:30:04.727185826Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1351","timestamp":"2022-08-12T13:30:04.727058523Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1359","timestamp":"2022-08-12T13:30:04.727233736Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1352","timestamp":"2022-08-12T13:30:04.727059973Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1360","timestamp":"2022-08-12T13:30:04.727235228Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1353","timestamp":"2022-08-12T13:30:04.727071812Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1354","timestamp":"2022-08-12T13:30:04.727096108Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1355","timestamp":"2022-08-12T13:30:04.727138971Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1356","timestamp":"2022-08-12T13:30:04.727140372Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1361","timestamp":"2022-08-12T13:30:04.727247023Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1362","timestamp":"2022-08-12T13:30:04.727270416Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1363","timestamp":"2022-08-12T13:30:04.727314473Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1364","timestamp":"2022-08-12T13:30:04.72731596Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1365","timestamp":"2022-08-12T13:30:04.727327822Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1366","timestamp":"2022-08-12T13:30:04.727350628Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1369","timestamp":"2022-08-12T13:30:04.727410743Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1367","timestamp":"2022-08-12T13:30:04.727395569Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1370","timestamp":"2022-08-12T13:30:04.727434352Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1368","timestamp":"2022-08-12T13:30:04.727397165Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1371","timestamp":"2022-08-12T13:30:04.727474503Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1372","timestamp":"2022-08-12T13:30:04.727475918Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1373","timestamp":"2022-08-12T13:30:04.727488454Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1374","timestamp":"2022-08-12T13:30:04.727512885Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1375","timestamp":"2022-08-12T13:30:04.727554338Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1376","timestamp":"2022-08-12T13:30:04.727555746Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1377","timestamp":"2022-08-12T13:30:04.727566931Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1378","timestamp":"2022-08-12T13:30:04.727590862Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1379","timestamp":"2022-08-12T13:30:04.727651378Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1380","timestamp":"2022-08-12T13:30:04.727653445Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1381","timestamp":"2022-08-12T13:30:04.72766453Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1382","timestamp":"2022-08-12T13:30:04.727693605Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1383","timestamp":"2022-08-12T13:30:04.727757231Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1384","timestamp":"2022-08-12T13:30:04.727758725Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1385","timestamp":"2022-08-12T13:30:04.727768967Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1386","timestamp":"2022-08-12T13:30:04.727792182Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1387","timestamp":"2022-08-12T13:30:04.727855435Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1388","timestamp":"2022-08-12T13:30:04.727856854Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1389","timestamp":"2022-08-12T13:30:04.72786886Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1390","timestamp":"2022-08-12T13:30:04.7278914Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1391","timestamp":"2022-08-12T13:30:04.727956863Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1392","timestamp":"2022-08-12T13:30:04.72795815Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1393","timestamp":"2022-08-12T13:30:04.727971965Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1394","timestamp":"2022-08-12T13:30:04.728004329Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1395","timestamp":"2022-08-12T13:30:04.728119497Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1401","timestamp":"2022-08-12T13:30:04.728394825Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1396","timestamp":"2022-08-12T13:30:04.728121353Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1402","timestamp":"2022-08-12T13:30:04.728444298Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1403","timestamp":"2022-08-12T13:30:04.728538214Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1397","timestamp":"2022-08-12T13:30:04.728134505Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1404","timestamp":"2022-08-12T13:30:04.728539818Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1398","timestamp":"2022-08-12T13:30:04.728185821Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1399","timestamp":"2022-08-12T13:30:04.72837971Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1400","timestamp":"2022-08-12T13:30:04.728381238Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1405","timestamp":"2022-08-12T13:30:04.72855015Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1406","timestamp":"2022-08-12T13:30:04.728581422Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1407","timestamp":"2022-08-12T13:30:04.728676286Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1408","timestamp":"2022-08-12T13:30:04.72867775Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1409","timestamp":"2022-08-12T13:30:04.72869072Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1410","timestamp":"2022-08-12T13:30:04.728714949Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1411","timestamp":"2022-08-12T13:30:04.728826928Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1412","timestamp":"2022-08-12T13:30:04.728828315Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1413","timestamp":"2022-08-12T13:30:04.728839654Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1414","timestamp":"2022-08-12T13:30:04.728873478Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1415","timestamp":"2022-08-12T13:30:04.728953805Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1416","timestamp":"2022-08-12T13:30:04.728955354Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1417","timestamp":"2022-08-12T13:30:04.728966312Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1418","timestamp":"2022-08-12T13:30:04.72900211Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1419","timestamp":"2022-08-12T13:30:04.729084668Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1421","timestamp":"2022-08-12T13:30:04.729100937Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1420","timestamp":"2022-08-12T13:30:04.729086148Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1422","timestamp":"2022-08-12T13:30:04.72912988Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1423","timestamp":"2022-08-12T13:30:04.729222323Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1424","timestamp":"2022-08-12T13:30:04.729223759Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1425","timestamp":"2022-08-12T13:30:04.72923462Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1426","timestamp":"2022-08-12T13:30:04.729260529Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1427","timestamp":"2022-08-12T13:30:04.729324011Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1428","timestamp":"2022-08-12T13:30:04.729327645Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1429","timestamp":"2022-08-12T13:30:04.729357329Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1430","timestamp":"2022-08-12T13:30:04.729400301Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1433","timestamp":"2022-08-12T13:30:04.729499517Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1431","timestamp":"2022-08-12T13:30:04.729480967Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1434","timestamp":"2022-08-12T13:30:04.729542358Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1432","timestamp":"2022-08-12T13:30:04.729483589Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1435","timestamp":"2022-08-12T13:30:04.729591928Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1436","timestamp":"2022-08-12T13:30:04.72959343Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1437","timestamp":"2022-08-12T13:30:04.729610379Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1438","timestamp":"2022-08-12T13:30:04.729637655Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1441","timestamp":"2022-08-12T13:30:04.729739369Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1439","timestamp":"2022-08-12T13:30:04.729715463Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1445","timestamp":"2022-08-12T13:30:04.729873393Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1440","timestamp":"2022-08-12T13:30:04.729716956Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1446","timestamp":"2022-08-12T13:30:04.729911713Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1447","timestamp":"2022-08-12T13:30:04.730000177Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1448","timestamp":"2022-08-12T13:30:04.730002293Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1442","timestamp":"2022-08-12T13:30:04.729771376Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1443","timestamp":"2022-08-12T13:30:04.729850571Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1444","timestamp":"2022-08-12T13:30:04.729853238Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1449","timestamp":"2022-08-12T13:30:04.730014324Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1450","timestamp":"2022-08-12T13:30:04.730040765Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1451","timestamp":"2022-08-12T13:30:04.730101644Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1452","timestamp":"2022-08-12T13:30:04.730104439Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1453","timestamp":"2022-08-12T13:30:04.730127053Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1454","timestamp":"2022-08-12T13:30:04.730164471Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1455","timestamp":"2022-08-12T13:30:04.730231814Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1456","timestamp":"2022-08-12T13:30:04.730233141Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1461","timestamp":"2022-08-12T13:30:04.730397999Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1457","timestamp":"2022-08-12T13:30:04.730245355Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1462","timestamp":"2022-08-12T13:30:04.730423389Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1458","timestamp":"2022-08-12T13:30:04.730280749Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1463","timestamp":"2022-08-12T13:30:04.730515122Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1459","timestamp":"2022-08-12T13:30:04.730385717Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1464","timestamp":"2022-08-12T13:30:04.730516913Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1465","timestamp":"2022-08-12T13:30:04.730529857Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1466","timestamp":"2022-08-12T13:30:04.730562975Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1467","timestamp":"2022-08-12T13:30:04.730634337Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1468","timestamp":"2022-08-12T13:30:04.730635776Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1460","timestamp":"2022-08-12T13:30:04.730387184Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1469","timestamp":"2022-08-12T13:30:04.730648601Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1470","timestamp":"2022-08-12T13:30:04.730673802Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1471","timestamp":"2022-08-12T13:30:04.730745535Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1472","timestamp":"2022-08-12T13:30:04.73074692Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1473","timestamp":"2022-08-12T13:30:04.730757894Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1474","timestamp":"2022-08-12T13:30:04.730805758Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1475","timestamp":"2022-08-12T13:30:04.730872523Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1476","timestamp":"2022-08-12T13:30:04.73087409Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1477","timestamp":"2022-08-12T13:30:04.730896539Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1478","timestamp":"2022-08-12T13:30:04.730943956Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1479","timestamp":"2022-08-12T13:30:04.731014997Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1480","timestamp":"2022-08-12T13:30:04.731017251Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1481","timestamp":"2022-08-12T13:30:04.73107221Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1482","timestamp":"2022-08-12T13:30:04.731116021Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1483","timestamp":"2022-08-12T13:30:04.731185458Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1484","timestamp":"2022-08-12T13:30:04.731187002Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1485","timestamp":"2022-08-12T13:30:04.731197755Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1486","timestamp":"2022-08-12T13:30:04.731238979Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1493","timestamp":"2022-08-12T13:30:04.731428694Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1487","timestamp":"2022-08-12T13:30:04.73131753Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1494","timestamp":"2022-08-12T13:30:04.731468104Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1488","timestamp":"2022-08-12T13:30:04.73131909Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1495","timestamp":"2022-08-12T13:30:04.731529004Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1496","timestamp":"2022-08-12T13:30:04.731530374Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1489","timestamp":"2022-08-12T13:30:04.731329445Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1490","timestamp":"2022-08-12T13:30:04.731354836Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1491","timestamp":"2022-08-12T13:30:04.731416234Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1492","timestamp":"2022-08-12T13:30:04.731417833Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1497","timestamp":"2022-08-12T13:30:04.731540925Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1498","timestamp":"2022-08-12T13:30:04.731566333Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1499","timestamp":"2022-08-12T13:30:04.731622626Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} {"cluster":"infrastage","component":"stdout","container_id":"d4a92a653f5f","container_name":"main","hostname":"10-40-22-18-uswest1cdevc","instance":"test_output.foo","level":"debug","message":"1500","timestamp":"2022-08-12T13:30:04.731687734Z","pod_name":"compute-infra-test-service.test--output.228.foo.jawpvt","tron_run_number":228} ================================================ FILE: tests/utils/state_test.py ================================================ from testifycompat import assert_equal from testifycompat import setup from testifycompat import TestCase from tron.utils import state class TestStateMachineSimple(TestCase): @setup def build_machine(self): self.state_green = "green" self.state_red = "red" self.machine = state.Machine(self.state_red, red=dict(true="green")) def test_transition_many(self): # Stay the same assert not self.machine.transition("missing") assert_equal(self.machine.state, self.state_red) # Traffic has arrived self.machine.transition("true") assert_equal(self.machine.state, self.state_green) # Still traffic self.machine.transition("true") assert_equal(self.machine.state, self.state_green) def test_check(self): assert not self.machine.check(False) assert_equal(self.machine.check("true"), self.state_green) assert_equal(self.machine.state, self.state_red) class TestStateMachineMultiOption(TestCase): @setup def build_machine(self): # Generalized rules of a conversation # If they are talking, we should listen # If they are listening, we should talk # If they are ignoring us we should get angry self.machine = state.Machine( "listening", listening=dict(listening="talking"), talking=dict(ignoring="angry", talking="listening"), ) def test_transition_many(self): # Talking, we should listen self.machine.transition("talking") assert_equal(self.machine.state, "listening") # Now be polite self.machine.transition("listening") assert_equal(self.machine.state, "talking") self.machine.transition("listening") assert_equal(self.machine.state, "talking") # But they are tired of us... self.machine.transition("ignoring") assert_equal(self.machine.state, "angry") def test_transition_set(self): expected = {"listening", "talking", "ignoring"} assert_equal(set(self.machine.transition_names), expected) ================================================ FILE: tests/utils/timeutils_test.py ================================================ import datetime import pytz from testifycompat import assert_equal from testifycompat import setup from testifycompat import TestCase from tests import testingutils from tron.utils import timeutils from tron.utils.timeutils import DateArithmetic from tron.utils.timeutils import duration from tron.utils.timeutils import macro_timedelta class TestTimeDelta(TestCase): @setup def make_dates(self): self.start_nonleap = datetime.datetime(year=2011, month=1, day=1) self.end_nonleap = datetime.datetime(year=2011, month=12, day=31) self.begin_feb_nonleap = datetime.datetime(year=2011, month=2, day=1) self.start_leap = datetime.datetime(year=2012, month=1, day=1) self.end_leap = datetime.datetime(year=2012, month=12, day=31) self.begin_feb_leap = datetime.datetime(year=2012, month=2, day=1) def check_delta(self, start, target, years=0, months=0, days=0): assert_equal( start + macro_timedelta( start, years=years, months=months, days=days, ), target, ) def test_days(self): self.check_delta( self.start_nonleap, datetime.datetime(year=2011, month=1, day=11), days=10, ) self.check_delta( self.end_nonleap, datetime.datetime(year=2012, month=1, day=10), days=10, ) self.check_delta( self.start_leap, datetime.datetime(year=2012, month=1, day=11), days=10, ) self.check_delta( self.end_leap, datetime.datetime(year=2013, month=1, day=10), days=10, ) self.check_delta( self.begin_feb_nonleap, datetime.datetime(year=2011, month=3, day=1), days=28, ) self.check_delta( self.begin_feb_leap, datetime.datetime(year=2012, month=3, day=1), days=29, ) def test_months(self): self.check_delta( self.start_nonleap, datetime.datetime(year=2011, month=11, day=1), months=10, ) self.check_delta( self.end_nonleap, datetime.datetime(year=2012, month=10, day=31), months=10, ) self.check_delta( self.start_leap, datetime.datetime(year=2012, month=11, day=1), months=10, ) self.check_delta( self.end_leap, datetime.datetime(year=2013, month=10, day=31), months=10, ) self.check_delta( self.begin_feb_nonleap, datetime.datetime(year=2011, month=12, day=1), months=10, ) self.check_delta( self.begin_feb_leap, datetime.datetime(year=2012, month=12, day=1), months=10, ) def test_years(self): self.check_delta( self.start_nonleap, datetime.datetime(year=2015, month=1, day=1), years=4, ) self.check_delta( self.end_nonleap, datetime.datetime(year=2015, month=12, day=31), years=4, ) self.check_delta( self.start_leap, datetime.datetime(year=2016, month=1, day=1), years=4, ) self.check_delta( self.end_leap, datetime.datetime(year=2016, month=12, day=31), years=4, ) self.check_delta( self.begin_feb_nonleap, datetime.datetime(year=2015, month=2, day=1), years=4, ) self.check_delta( self.begin_feb_leap, datetime.datetime(year=2016, month=2, day=1), years=4, ) def test_start_date_with_timezone(self): pacific_tz = pytz.timezone("US/Pacific") start_date = pacific_tz.localize( datetime.datetime(year=2018, month=1, day=3, hour=13), ) expected_end = pacific_tz.localize( datetime.datetime(year=2018, month=1, day=1, hour=13), ) self.check_delta( start_date, expected_end, days=-2, ) class TestDuration(TestCase): @setup def setup_times(self): self.earliest = datetime.datetime(2012, 2, 1, 3, 0, 0) self.latest = datetime.datetime(2012, 2, 1, 3, 20, 0) def test_duration(self): assert_equal( duration(self.earliest, self.latest), datetime.timedelta(0, 60 * 20), ) def test_duration_no_end(self): delta = duration(self.earliest) assert delta.days >= 40 def test_duration_no_start(self): assert_equal(duration(None), None) class TestDeltaTotalSeconds(TestCase): def test(self): expected = 86702.004002999995 delta = datetime.timedelta(*range(1, 6)) delta_seconds = timeutils.delta_total_seconds(delta) assert_equal(delta_seconds, expected) class DateArithmeticTestCase(testingutils.MockTimeTestCase): # Set a date with days less then 28, otherwise some tests will fail # when run on days > 28. now = datetime.datetime(2012, 3, 20) def _cmp_date(self, item, dt): assert_equal(DateArithmetic.parse(item), dt.strftime("%Y-%m-%d")) def _cmp_day(self, item, dt): assert_equal(DateArithmetic.parse(item), dt.strftime("%d")) def _cmp_month(self, item, dt): assert_equal(DateArithmetic.parse(item), dt.strftime("%m")) def _cmp_year(self, item, dt): assert_equal(DateArithmetic.parse(item), dt.strftime("%Y")) def test_shortdate(self): self._cmp_date("shortdate", self.now) def test_shortdate_plus(self): for i in range(50): dt = self.now + datetime.timedelta(days=i) self._cmp_date("shortdate+%s" % i, dt) def test_shortdate_minus(self): for i in range(50): dt = self.now - datetime.timedelta(days=i) self._cmp_date("shortdate-%s" % i, dt) def test_day(self): self._cmp_day("day", self.now) def test_day_minus(self): for i in range(50): dt = self.now - datetime.timedelta(days=i) self._cmp_day("day-%s" % i, dt) def test_day_plus(self): for i in range(50): dt = self.now + datetime.timedelta(days=i) self._cmp_day("day+%s" % i, dt) def test_month(self): self._cmp_month("month", self.now) def test_month_plus(self): for i in range(50): dt = self.now + timeutils.macro_timedelta(self.now, months=i) self._cmp_month("month+%s" % i, dt) def test_month_minus(self): for i in range(50): dt = self.now - timeutils.macro_timedelta(self.now, months=i) self._cmp_month("month-%s" % i, dt) def test_year(self): self._cmp_year("year", self.now) def test_year_plus(self): for i in range(50): dt = self.now + timeutils.macro_timedelta(self.now, years=i) self._cmp_year("year+%s" % i, dt) def test_year_minus(self): for i in range(50): dt = self.now - timeutils.macro_timedelta(self.now, years=i) self._cmp_year("year-%s" % i, dt) def test_unixtime(self): timestamp = int(self.now.timestamp()) assert_equal(DateArithmetic.parse("unixtime"), timestamp) def test_unixtime_plus(self): timestamp = int(self.now.timestamp()) + 100 assert_equal(DateArithmetic.parse("unixtime+100"), timestamp) def test_unixtime_minus(self): timestamp = int(self.now.timestamp()) - 99 assert_equal(DateArithmetic.parse("unixtime-99"), timestamp) def test_daynumber(self): daynum = self.now.toordinal() assert_equal(DateArithmetic.parse("daynumber"), daynum) def test_daynumber_plus(self): daynum = self.now.toordinal() + 1 assert_equal(DateArithmetic.parse("daynumber+1"), daynum) def test_daynumber_minus(self): daynum = self.now.toordinal() - 1 assert_equal(DateArithmetic.parse("daynumber-1"), daynum) def test_hour(self): hour = self.now.strftime("%H") assert_equal(DateArithmetic.parse("hour"), hour) def test_hour_plus(self): hour = "%02d" % ((int(self.now.strftime("%H")) + 1) % 24) assert_equal(DateArithmetic.parse("hour+1"), hour) def test_hour_minus(self): hour = "%02d" % ((int(self.now.strftime("%H")) - 1) % 24) assert_equal(DateArithmetic.parse("hour-1"), hour) def test_bad_date_format(self): assert DateArithmetic.parse("~~") is None def test_round_day(self): start = datetime.datetime(2019, 3, 30) delta = timeutils.macro_timedelta(start, months=-1) assert (start + delta).day == 28 class DateArithmeticYMDHTest(TestCase): def test_ym_plus(self): def parse(*ym): return DateArithmetic.parse("ym+1", datetime.datetime(*ym)) assert_equal(parse(2018, 1, 1), "2018-02") assert_equal(parse(2017, 12, 1), "2018-01") def test_ym_minus(self): def parse(*ym): return DateArithmetic.parse("ym-1", datetime.datetime(*ym)) assert_equal(parse(2018, 1, 1), "2017-12") assert_equal(parse(2018, 2, 1), "2018-01") def test_ymd_plus(self): def parse(*ymd): return DateArithmetic.parse("ymd+1", datetime.datetime(*ymd)) assert_equal(parse(2018, 1, 1), "2018-01-02") assert_equal(parse(2018, 1, 31), "2018-02-01") def test_ymd_minus(self): def parse(*ymd): return DateArithmetic.parse("ymd-1", datetime.datetime(*ymd)) assert_equal(parse(2018, 1, 1), "2017-12-31") assert_equal(parse(2018, 1, 2), "2018-01-01") def test_ymdh_plus(self): def parse(*ymdh): return DateArithmetic.parse("ymdh+1", datetime.datetime(*ymdh)) assert_equal(parse(2018, 1, 1, 1), "2018-01-01T02") assert_equal(parse(2018, 1, 31, 23), "2018-02-01T00") def test_ymdh_minus(self): def parse(*ymdh): return DateArithmetic.parse("ymdh-1", datetime.datetime(*ymdh)) assert_equal(parse(2018, 1, 1, 1), "2018-01-01T00") assert_equal(parse(2018, 1, 1, 0), "2017-12-31T23") def test_ymdhm_plus(self): def parse(*ymdhm): return DateArithmetic.parse("ymdhm+1", datetime.datetime(*ymdhm)) assert_equal(parse(2018, 1, 1, 1, 1), "2018-01-01T01:02") assert_equal(parse(2018, 1, 31, 23, 59), "2018-02-01T00:00") def test_ymdhm_minus(self): def parse(*ymdhm): return DateArithmetic.parse("ymdhm-1", datetime.datetime(*ymdhm)) assert_equal(parse(2018, 1, 1, 1, 2), "2018-01-01T01:01") assert_equal(parse(2018, 1, 1, 0, 0), "2017-12-31T23:59") def test_ym_minus_round(self): dt = datetime.datetime(2019, 3, 30) s = timeutils.DateArithmetic.parse("ym-1", dt=dt) assert s == "2019-02" def test_ymd_plus_whitespace(self): def parse(*ymd): return DateArithmetic.parse("ymd + 1", datetime.datetime(*ymd)) assert_equal(parse(2018, 1, 1), "2018-01-02") assert_equal(parse(2018, 1, 31), "2018-02-01") def test_ymd_minus_whitespace(self): def parse(*ymd): return DateArithmetic.parse("ymd- 1", datetime.datetime(*ymd)) assert_equal(parse(2018, 1, 1), "2017-12-31") assert_equal(parse(2018, 1, 2), "2018-01-01") class TestDateArithmeticWithTimezone(DateArithmeticTestCase): now = pytz.timezone("US/Pacific").localize(datetime.datetime(2012, 3, 20)) ================================================ FILE: tests/utils/trontimespec_test.py ================================================ import datetime import pytz from testifycompat import assert_equal from testifycompat import run from testifycompat import TestCase from tron.utils import trontimespec class TestGetTime(TestCase): def test_get_time(self): assert_equal(datetime.time(4, 15), trontimespec.get_time("4:15")) assert_equal(datetime.time(22, 59), trontimespec.get_time("22:59")) def test_get_time_invalid_time(self): assert not trontimespec.get_time("25:00") assert not trontimespec.get_time("22:61") class TestTimeSpecification(TestCase): def _cmp(self, start_time, expected): start_time = datetime.datetime(*start_time) expected = datetime.datetime(*expected) assert_equal(self.time_spec.get_match(start_time), expected) def test_get_match_months(self): self.time_spec = trontimespec.TimeSpecification(months=[1, 5]) self._cmp((2012, 3, 14), (2012, 5, 1)) self._cmp((2012, 5, 22), (2012, 5, 23)) self._cmp((2012, 12, 22), (2013, 1, 1)) def test_get_match_monthdays(self): self.time_spec = trontimespec.TimeSpecification( monthdays=[10, 3, 3, 10], ) self._cmp((2012, 3, 14), (2012, 4, 3)) self._cmp((2012, 3, 1), (2012, 3, 3)) def test_get_match_weekdays(self): self.time_spec = trontimespec.TimeSpecification(weekdays=[2, 3]) self._cmp((2012, 3, 14), (2012, 3, 20)) self._cmp((2012, 3, 20), (2012, 3, 21)) def test_next_month_generator(self): time_spec = trontimespec.TimeSpecification(months=[2, 5]) gen = time_spec.next_month(datetime.datetime(2012, 3, 14)) expected = [(5, 2012), (2, 2013), (5, 2013), (2, 2014)] assert_equal([next(gen) for _ in range(4)], expected) def test_next_day_monthdays(self): time_spec = trontimespec.TimeSpecification(monthdays=[5, 10, 15]) gen = time_spec.next_day(14, 2012, 3) assert_equal(list(gen), [15]) gen = time_spec.next_day(1, 2012, 3) assert_equal(list(gen), [5, 10, 15]) def test_next_day_monthdays_with_last(self): time_spec = trontimespec.TimeSpecification(monthdays=[5, "LAST"]) gen = time_spec.next_day(14, 2012, 3) assert_equal(list(gen), [31]) def test_next_day_weekdays(self): time_spec = trontimespec.TimeSpecification(weekdays=[1, 5]) gen = time_spec.next_day(14, 2012, 3) assert_equal(list(gen), [16, 19, 23, 26, 30]) gen = time_spec.next_day(1, 2012, 3) assert_equal(list(gen), [2, 5, 9, 12, 16, 19, 23, 26, 30]) def test_next_day_weekdays_with_ordinals(self): time_spec = trontimespec.TimeSpecification( weekdays=[1, 5], ordinals=[1, 3], ) gen = time_spec.next_day(14, 2012, 3) assert_equal(list(gen), [16, 19]) gen = time_spec.next_day(1, 2012, 3) assert_equal(list(gen), [2, 5, 16, 19]) def test_next_time_timestr(self): time_spec = trontimespec.TimeSpecification(timestr="13:13") start_date = datetime.datetime(2012, 3, 14, 0, 15) time = time_spec.next_time(start_date, True) assert_equal(time, datetime.time(13, 13)) start_date = datetime.datetime(2012, 3, 14, 13, 13) assert time_spec.next_time(start_date, True) is None time = time_spec.next_time(start_date, False) assert_equal(time, datetime.time(13, 13)) def test_next_time_hours(self): time_spec = trontimespec.TimeSpecification(hours=[4, 10]) start_date = datetime.datetime(2012, 3, 14, 0, 15) time = time_spec.next_time(start_date, True) assert_equal(time, datetime.time(4, 0)) start_date = datetime.datetime(2012, 3, 14, 13, 13) assert time_spec.next_time(start_date, True) is None time = time_spec.next_time(start_date, False) assert_equal(time, datetime.time(4, 0)) def test_next_time_minutes(self): time_spec = trontimespec.TimeSpecification( minutes=[30, 20, 30], seconds=[0], ) start_date = datetime.datetime(2012, 3, 14, 0, 25) time = time_spec.next_time(start_date, True) assert_equal(time, datetime.time(0, 30)) start_date = datetime.datetime(2012, 3, 14, 23, 30) assert time_spec.next_time(start_date, True) is None time = time_spec.next_time(start_date, False) assert_equal(time, datetime.time(0, 20)) def test_next_time_hours_and_minutes_and_seconds(self): time_spec = trontimespec.TimeSpecification( minutes=[20, 30], hours=[1, 5], seconds=[4, 5], ) start_date = datetime.datetime(2012, 3, 14, 1, 25) time = time_spec.next_time(start_date, True) assert_equal(time, datetime.time(1, 30, 4)) start_date = datetime.datetime(2012, 3, 14, 5, 30, 6) assert time_spec.next_time(start_date, True) is None time = time_spec.next_time(start_date, False) assert_equal(time, datetime.time(1, 20, 4)) def test_get_match_dst_spring_forward(self): tz = pytz.timezone("US/Pacific") time_spec = trontimespec.TimeSpecification( hours=[0, 1, 2, 3, 4], minutes=[0], seconds=[0], timezone="US/Pacific", ) start = trontimespec.naive_as_timezone(datetime.datetime(2020, 3, 8, 1), tz) # Springing forward, the next hour after 1AM should be 3AM next_time = time_spec.get_match(start) assert next_time.hour == 3 def test_get_match_dst_fall_back(self): tz = pytz.timezone("US/Pacific") time_spec = trontimespec.TimeSpecification( hours=[0, 1, 2, 3, 4], minutes=[0], seconds=[0], timezone="US/Pacific", ) start = trontimespec.naive_as_timezone(datetime.datetime(2020, 11, 1, 1), tz) # Falling back, the next hour after 1AM is 1AM again. But we only run on the first 1AM # Next run time should be 2AM next_time = time_spec.get_match(start) assert next_time.hour == 2 if __name__ == "__main__": run() ================================================ FILE: tools/action_dag_diagram.py ================================================ """ Create a graphviz diagram from a Tron Job configuration. Usage: python tools/action_dag_diagram.py -c -n This will create a file named .dot You can create a diagram using: dot -Tpng -o .png .dot """ import optparse from tron.config import manager from tron.config import schema def parse_args(): parser = optparse.OptionParser() parser.add_option("-c", "--config", help="Tron configuration path.") parser.add_option( "-n", "--name", help="Job name to graph. Also used as output filename.", ) parser.add_option( "--namespace", default=schema.MASTER_NAMESPACE, help="Configuration namespace which contains the job.", ) opts, _ = parser.parse_args() if not opts.config: parser.error("A config filename is required.") if not opts.name: parser.error("A Job name is required.") return opts def build_diagram(job_config): edges, nodes = [], [] for action in job_config.actions.values(): shape = "invhouse" if not action.requires else "rect" nodes.append(f"node [shape = {shape}]; {action.name}") for required_action in action.requires: edges.append(f"{required_action} -> {action.name}") return "digraph g{{{}\n{}}}".format("\n".join(nodes), "\n".join(edges)) def get_job(config_container, namespace, job_name): if namespace not in config_container: raise ValueError("Unknown namespace: %s" % namespace) config = config_container[opts.namespace] if job_name not in config.jobs: raise ValueError("Could not find Job %s" % job_name) return config.jobs[job_name] if __name__ == "__main__": opts = parse_args() config_manager = manager.ConfigManager(opts.config) container = config_manager.load() job_config = get_job(container, opts.namespace, opts.name) graph = build_diagram(job_config) with open("%s.dot" % opts.name, "w") as fh: fh.write(graph) ================================================ FILE: tools/compress_json.py ================================================ import argparse import gzip import math import os import sys import threading import time from concurrent.futures import as_completed from concurrent.futures import ThreadPoolExecutor import boto3 from boto3.resources.base import ServiceResource from tron.core.job import Job from tron.core.jobrun import JobRun from tron.serialize import runstate # Max DynamoDB object size is 400KB. Since we save two copies of the object (pickled and JSON), # we need to consider this max size applies to the entire item, so we use a max size of 200KB # for each version. OBJECT_SIZE = 150_000 # DynamoDB TransactWriteItems supports up to 100 items per call. We use 25 # which comfortably fits under the 4MB request size limit (25 × 150KB = 3.75MB # worst case). This is sufficient to handle our largest items (~20 partitions # uncompressed) in a single atomic transaction after compression. MAX_TRANSACT_WRITE_ITEMS = 25 def get_dynamodb_table( aws_profile: str | None = None, table: str = "infrastage-tron-state", region: str = "us-west-1" ) -> ServiceResource: session = boto3.Session(profile_name=aws_profile) if aws_profile else boto3.Session() return session.resource("dynamodb", region_name=region).Table(table) def get_dynamodb_client( aws_profile: str | None = None, region: str = "us-west-1", ): session = boto3.Session(profile_name=aws_profile) if aws_profile else boto3.Session() return session.client("dynamodb", region_name=region) def scan_keys(source_table: ServiceResource) -> set[str]: """Streaming scan that only projects the key attribute and collects unique partition keys. Never stores full items in memory — only the 'key' strings. """ unique_keys: set[str] = set() scan_kwargs = { "ProjectionExpression": "#k", "ExpressionAttributeNames": {"#k": "key"}, } response = source_table.scan(**scan_kwargs) for item in response.get("Items", []): unique_keys.add(item.get("key", "Unknown Key")) while "LastEvaluatedKey" in response: response = source_table.scan(ExclusiveStartKey=response["LastEvaluatedKey"], **scan_kwargs) for item in response.get("Items", []): unique_keys.add(item.get("key", "Unknown Key")) return unique_keys def resolve_keys(args, parser, source_table: ServiceResource) -> list[str]: if not args.keys and not args.keys_file and not args.all: parser.error("You must provide either --keys, --keys-file, or --all.") if args.all: print("Scanning table for all keys (keys-only projection)...") keys_set = scan_keys(source_table) print(f"Found {len(keys_set)} unique keys.") return list(keys_set) keys = [] if args.keys: keys.extend(args.keys) if args.keys_file: try: with open(args.keys_file) as f: keys_from_file = [line.strip() for line in f if line.strip()] keys.extend(keys_from_file) except Exception as e: parser.error(f"Error reading keys from file {args.keys_file}: {e}") if not keys: parser.error("No keys provided. Please provide keys via --keys or --keys-file.") return list(set(keys)) def is_compressed(json_val) -> bool: """Check if a json_val from DynamoDB is compressed (Binary type) vs uncompressed (String type). boto3 high-level resource API returns: - "S" type as Python str - "B" type as boto3.dynamodb.types.Binary (which wraps bytes and has a .value attribute) """ if hasattr(json_val, "value"): # boto3.dynamodb.types.Binary return True if isinstance(json_val, bytes): # I don't think this is possible in the high-level API, but it's harmless to check return True return False def get_json_val_bytes(json_val) -> bytes: """Extract raw bytes from a json_val, whether it's a Binary wrapper or raw bytes.""" if hasattr(json_val, "value"): return bytes(json_val.value) if isinstance(json_val, bytes): return bytes(json_val) raise TypeError(f"Unexpected json_val type: {type(json_val)}") def classify_item(item: dict) -> str: """Classify a partition-0 item into one of: compressed, uncompressed, pickle_only, no_data.""" has_json = "json_val" in item has_pickle = "val" in item if has_json: if is_compressed(item["json_val"]): return "compressed" else: return "uncompressed" elif has_pickle: # Shouldn't exist. If we have items with only pickle data we want to know return "pickle_only" else: # Shouldn't exist. If we have funky items with no json_val/val we want to know return "no_data" def compress_json_for_key( source_table: ServiceResource, client, table_name: str, key: str, dry_run: bool = True ) -> str: """Compress uncompressed JSON for a single key. Reads all json_val partitions via get_item (ConsistentRead), gzip-compresses the combined JSON, and writes the compressed data back using TransactWriteItems with ConditionExpressions to guard against concurrent trond writes. Returns a status string: "compressed", "already_compressed", "no_json", "skipped", "concurrent_update", or raises on error (including throttle exhaustion). """ response = source_table.get_item(Key={"key": key, "index": 0}, ConsistentRead=True) if "Item" not in response: print(f" SKIP (no item found): {key}") return "skipped" item_0 = response["Item"] if "json_val" not in item_0: print(f" SKIP (no json_val — needs pickles_to_json.py first): {key}") return "no_json" json_val = item_0["json_val"] if is_compressed(json_val): print(f" SKIP (already compressed): {key}") return "already_compressed" # It's an uncompressed string — collect all partitions via get_item num_json_partitions = int(item_0.get("num_json_val_partitions", 1)) combined_json = "" for index in range(num_json_partitions): if index == 0: partition_item = item_0 else: resp = source_table.get_item(Key={"key": key, "index": index}, ConsistentRead=True) if "Item" not in resp: raise Exception(f"Missing JSON partition {index} for key {key}") partition_item = resp["Item"] if "json_val" not in partition_item: raise Exception(f"No 'json_val' in partition {index} for key {key}") combined_json += partition_item["json_val"] # Validate JSON round-trips before compressing state_type = key.split()[0] if state_type == runstate.JOB_STATE: Job.from_json(combined_json) elif state_type == runstate.JOB_RUN_STATE: JobRun.from_json(combined_json) else: print(f" SKIP (unknown state type '{state_type}'): {key}") return "skipped" # Compress compressed = gzip.compress(combined_json.encode("utf-8")) num_compressed_partitions = math.ceil(len(compressed) / OBJECT_SIZE) original_size = len(combined_json.encode("utf-8")) compressed_size = len(compressed) ratio = (1 - compressed_size / original_size) * 100 if original_size > 0 else 0 if dry_run: print( f" DRY RUN (would compress): {key} " f"({original_size:,} bytes -> {compressed_size:,} bytes, {ratio:.1f}% reduction, " f"{num_json_partitions} partitions -> {num_compressed_partitions} partitions)" ) else: # Build TransactWriteItems with conditional expressions. # Conditions on partition 0 ensure the item is still uncompressed and has the same # number of partitions we read — if trond wrote concurrently the condition fails. transact_items = [] for i in range(num_compressed_partitions): chunk = compressed[i * OBJECT_SIZE : (i + 1) * OBJECT_SIZE] update = { "Update": { "TableName": table_name, "Key": { "key": {"S": key}, "index": {"N": str(i)}, }, "UpdateExpression": "SET json_val = :json, num_json_val_partitions = :n", "ExpressionAttributeValues": { ":json": {"B": chunk}, ":n": {"N": str(num_compressed_partitions)}, }, }, } if i == 0: # Condition on partition 0: item must still be uncompressed with expected partition count update["Update"]["ConditionExpression"] = ( "attribute_exists(json_val) " "AND attribute_type(json_val, :string_type) " "AND num_json_val_partitions = :expected_partitions" ) update["Update"]["ExpressionAttributeValues"][":string_type"] = {"S": "S"} update["Update"]["ExpressionAttributeValues"][":expected_partitions"] = {"N": str(num_json_partitions)} transact_items.append(update) # Clean up excess partitions (REMOVE json_val where compressed needs fewer partitions) for i in range(num_compressed_partitions, num_json_partitions): transact_items.append( { "Update": { "TableName": table_name, "Key": { "key": {"S": key}, "index": {"N": str(i)}, }, "UpdateExpression": "REMOVE json_val", }, } ) if len(transact_items) > MAX_TRANSACT_WRITE_ITEMS: raise Exception( f"Compression requires {len(transact_items)} transaction items for key {key}, " f"exceeding single-transaction limit of {MAX_TRANSACT_WRITE_ITEMS}. " f"({num_json_partitions} uncompressed partitions, " f"{num_compressed_partitions} compressed partitions)" ) if compressed_size > 3_500_000: raise Exception( f"Compressed data too large ({compressed_size:,} bytes) for key {key} — " f"risks exceeding 4MB TransactWriteItems request limit." ) # Single atomic transaction — all updates and removes in one call. We retry # on ThrottlingException with exponential backoff on top of boto3's built-in # retries. During testing we are getting throttled almost immediately (on hot # keys) and quickly exhausting the default retry budget. This slows things # down a lot but allows the process to complete without manual intervention # or re-runs. max_retries = 5 for attempt in range(max_retries + 1): try: client.transact_write_items(TransactItems=transact_items) break except client.exceptions.TransactionCanceledException as e: reasons = e.response.get("CancellationReasons", []) if any(r.get("Code") in ("ConditionalCheckFailed", "TransactionConflict") for r in reasons): print(f" SKIPPED (concurrent update detected): {key}") return "concurrent_update" # Check if cancellation was due to throttling if any(r.get("Code") == "ThrottlingError" for r in reasons): if attempt < max_retries: wait = 2**attempt print(f" THROTTLED (attempt {attempt + 1}/{max_retries + 1}, retrying in {wait}s): {key}") time.sleep(wait) continue raise except Exception as e: if "ThrottlingException" in str(e) or "Throughput exceeds" in str(e): if attempt < max_retries: wait = 2**attempt print(f" THROTTLED (attempt {attempt + 1}/{max_retries + 1}, retrying in {wait}s): {key}") time.sleep(wait) continue raise else: raise Exception(f"Throttled after {max_retries + 1} attempts") print( f" COMPRESSED: {key} " f"({original_size:,} bytes -> {compressed_size:,} bytes, {ratio:.1f}% reduction, " f"{num_json_partitions} partitions -> {num_compressed_partitions} partitions)" ) return "compressed" def verify_compressed_json_for_key(source_table: ServiceResource, key: str) -> bool: """Re-read all compressed JSON partitions, gunzip, and validate via from_json. Returns True if verification succeeds, False otherwise. Prints the reason on failure. """ response = source_table.get_item(Key={"key": key, "index": 0}, ConsistentRead=True) if "Item" not in response: print(f" VERIFY FAIL (no item found): {key}") return False item_0 = response["Item"] if "json_val" not in item_0: print(f" VERIFY FAIL (no json_val): {key}") return False if not is_compressed(item_0["json_val"]): print(f" VERIFY FAIL (json_val is not compressed): {key}") return False num_json_partitions = int(item_0.get("num_json_val_partitions", 1)) # Reassemble compressed bytes from all partitions compressed_data = bytearray() for index in range(num_json_partitions): if index == 0: partition_item = item_0 else: resp = source_table.get_item(Key={"key": key, "index": index}, ConsistentRead=True) if "Item" not in resp: print(f" VERIFY FAIL (missing partition {index}): {key}") return False partition_item = resp["Item"] if "json_val" not in partition_item: print(f" VERIFY FAIL (no json_val in partition {index}): {key}") return False compressed_data += get_json_val_bytes(partition_item["json_val"]) # Decompress try: json_str = gzip.decompress(bytes(compressed_data)).decode("utf-8") except Exception as e: print(f" VERIFY FAIL (gunzip failed: {e}): {key}") return False # Validate via from_json state_type = key.split()[0] try: if state_type == runstate.JOB_STATE: Job.from_json(json_str) elif state_type == runstate.JOB_RUN_STATE: JobRun.from_json(json_str) else: print(f" VERIFY FAIL (unknown state type '{state_type}'): {key}") return False except Exception as e: print(f" VERIFY FAIL (from_json failed: {e}): {key}") return False return True def delete_pickle_for_key(source_table: ServiceResource, key: str, dry_run: bool = True) -> str: """Delete pickle data (val, num_partitions) for a single key. Verifies compressed JSON can be fully decoded and parsed before deleting. Returns a status string: "deleted", "refused", "no_pickle", "skipped", or raises on error. """ response = source_table.get_item(Key={"key": key, "index": 0}, ConsistentRead=True) if "Item" not in response: print(f" SKIP (no item found): {key}") return "skipped" item_0 = response["Item"] # Safety checks if "json_val" not in item_0: print(f" REFUSE (no json_val at all): {key}") return "refused" if not is_compressed(item_0["json_val"]): print(f" REFUSE (json_val is uncompressed — run 'compress' first): {key}") return "refused" if "val" not in item_0: print(f" SKIP (no pickle data to delete): {key}") return "no_pickle" # Verify compressed JSON is valid before deleting pickle if not verify_compressed_json_for_key(source_table, key): print(f" REFUSE (compressed JSON verification failed): {key}") return "refused" num_partitions = int(item_0.get("num_partitions", 1)) num_json_partitions = int(item_0.get("num_json_val_partitions", 1)) max_partitions = max(num_partitions, num_json_partitions) if dry_run: print( f" DRY RUN (would delete pickle): {key} ({num_partitions} pickle partitions across {max_partitions} items)" ) else: for i in range(max_partitions): source_table.update_item( Key={"key": key, "index": i}, UpdateExpression="REMOVE val, num_partitions", ) print(f" DELETED pickle: {key} ({num_partitions} pickle partitions removed across {max_partitions} items)") return "deleted" def cmd_compress(args, source_table: ServiceResource, client, table_name: str, keys: list[str]) -> None: dry_run = not args.execute workers = args.workers total = len(keys) counts = { "compressed": 0, "already_compressed": 0, "no_json": 0, "skipped": 0, "concurrent_update": 0, "failed": 0, } failed_keys = [] lock = threading.Lock() completed = [0] # mutable counter for progress mode = "DRY RUN" if dry_run else "EXECUTING" print(f"\n=== Compress JSON ({mode}, {workers} workers) ===") print(f"Processing {total} keys...\n") # Pre-create one Table resource per worker thread. The high-level # resource is not thread-safe, so each worker gets its own, but we # create them once upfront instead of per-key. thread_tables = [get_dynamodb_table(args.aws_profile, args.table_name, args.table_region) for _ in range(workers)] # Map thread IDs to table resources as workers claim them. thread_local = threading.local() def get_thread_table() -> ServiceResource: if not hasattr(thread_local, "table"): with lock: thread_local.table = thread_tables.pop() return thread_local.table def process_key(key: str) -> None: thread_table = get_thread_table() try: result = compress_json_for_key(thread_table, client, table_name, key, dry_run=dry_run) except Exception as e: result = "failed" with lock: failed_keys.append(key) print(f" FAILED ({key}): {e}") with lock: counts[result] += 1 completed[0] += 1 if completed[0] % 500 == 0 or completed[0] == total: print(f" Progress: {completed[0]}/{total} keys processed") sorted_keys = sorted(keys) with ThreadPoolExecutor(max_workers=workers) as pool: futures = {pool.submit(process_key, key): key for key in sorted_keys} for future in as_completed(futures): # Exceptions are already handled inside process_key, but catch # anything truly unexpected so one bad future doesn't kill the pool. try: future.result() except Exception as e: key = futures[future] print(f" UNEXPECTED ERROR ({key}): {e}") with lock: counts["failed"] += 1 failed_keys.append(key) print("\n=== Summary ===") print(f"Total keys: {total}") print(f"Compressed: {counts['compressed']}") print(f"Already compressed: {counts['already_compressed']}") print(f"No JSON (pickle-only):{counts['no_json']}") print(f"Skipped: {counts['skipped']}") print(f"Concurrent updates: {counts['concurrent_update']}") print(f"Failed: {counts['failed']}") if dry_run: print("\nDry run complete. No changes were made.") if args.failed_keys_output and failed_keys: with open(args.failed_keys_output, "w") as f: for key in failed_keys: f.write(f"{key}\n") print(f"Failed keys written to {args.failed_keys_output}") def cmd_delete_pickles(args, source_table: ServiceResource, keys: list[str]) -> None: dry_run = not args.execute if args.execute and not args.i_hereby_declare_we_no_longer_need_pickles: print( "ERROR: --execute for delete-pickles requires the safety flag:\n" " --i-hereby-declare-we-no-longer-need-pickles\n\n" "This operation is DESTRUCTIVE and IRREVERSIBLE. It removes all pickle data\n" "from DynamoDB items. Only proceed if you are certain that:\n" " 1. All items have valid compressed JSON (run 'status' to verify)\n" " 2. Tron is configured to read from JSON (read_json=True)\n" " 3. You have verified restores work from JSON on replica tables" ) sys.exit(1) total = len(keys) counts = {"deleted": 0, "refused": 0, "no_pickle": 0, "skipped": 0, "failed": 0} failed_keys = [] mode = "DRY RUN" if dry_run else "EXECUTING" print(f"\n=== Delete Pickles ({mode}) ===") print(f"Processing {total} keys...\n") for i, key in enumerate(sorted(keys), 1): print(f"[{i}/{total}] {key}") try: result = delete_pickle_for_key(source_table, key, dry_run=dry_run) counts[result] += 1 except Exception as e: print(f" FAILED: {e}") counts["failed"] += 1 failed_keys.append(key) print("\n=== Summary ===") print(f"Total keys: {total}") print(f"Deleted: {counts['deleted']}") print(f"Refused (no comp. JSON): {counts['refused']}") print(f"No pickle to delete: {counts['no_pickle']}") print(f"Skipped: {counts['skipped']}") print(f"Failed: {counts['failed']}") if dry_run: print("\nDry run complete. No changes were made.") if args.failed_keys_output and failed_keys: with open(args.failed_keys_output, "w") as f: for key in failed_keys: f.write(f"{key}\n") print(f"Failed keys written to {args.failed_keys_output}") def cmd_status(args, source_table: ServiceResource) -> None: print(f"\n=== Status: {args.table_name} ({args.table_region}) ===") print("Scanning partition-0 items...") counts = {"compressed": 0, "uncompressed": 0, "pickle_only": 0, "no_data": 0} total = 0 # Streaming scan filtered to partition 0 only, projecting just the attributes # needed for classification. This avoids per-key get_item calls. scan_kwargs = { "FilterExpression": "#idx = :zero", "ProjectionExpression": "#k, #idx, json_val, val", "ExpressionAttributeNames": {"#k": "key", "#idx": "index"}, "ExpressionAttributeValues": {":zero": 0}, } response = source_table.scan(**scan_kwargs) for item in response.get("Items", []): total += 1 counts[classify_item(item)] += 1 while "LastEvaluatedKey" in response: response = source_table.scan(ExclusiveStartKey=response["LastEvaluatedKey"], **scan_kwargs) for item in response.get("Items", []): total += 1 counts[classify_item(item)] += 1 print(f"\nTotal unique keys: {total:,}\n") for label, count_key in [ ("Compressed JSON (ready for pickle deletion)", "compressed"), ("Uncompressed JSON (needs compression)", "uncompressed"), ("Pickle only (anomalous, needs pickles_to_json.py)", "pickle_only"), ("No data (anomalous, needs investigation)", "no_data"), ]: count = counts[count_key] pct = (count / total * 100) if total > 0 else 0 print(f" {label + ':':<50} {count:>8,} ({pct:.1f}%)") def add_key_arguments(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "--keys", nargs="+", required=False, help="Specific key(s) to process.", ) subparser.add_argument( "--keys-file", required=False, help="Input file containing keys to process. One key per line.", ) subparser.add_argument( "--all", action="store_true", help="Process all keys in the table.", ) def main(): parser = argparse.ArgumentParser( description="Compress JSON and delete pickle data in Tron's DynamoDB state store.", epilog=""" Sub-commands: compress Compress uncompressed JSON ("S" type) to gzip-compressed binary ("B" type). delete-pickles Remove pickle data (val, num_partitions) from items that have compressed JSON. status Report the state of all keys in the table. Examples: Check status of a table: compress_json.py --table-name infrastage-tron-state --table-region us-west-1 status Dry-run compression for all keys: compress_json.py --table-name infrastage-tron-state --table-region us-west-1 compress --all Execute compression for specific keys: compress_json.py --table-name infrastage-tron-state --table-region us-west-1 compress --keys "job_state myjob" --execute Dry-run pickle deletion: compress_json.py --table-name infrastage-tron-state --table-region us-west-1 delete-pickles --all Execute pickle deletion (requires safety flag): compress_json.py --table-name infrastage-tron-state --table-region us-west-1 delete-pickles --all --execute --i-hereby-declare-we-no-longer-need-pickles """, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--aws-profile", default=os.environ.get("AWS_PROFILE", None), help="AWS profile to use (default: taken from AWS_PROFILE environment variable)", ) parser.add_argument("--table-name", required=True, help="Name of the DynamoDB table") parser.add_argument("--table-region", required=True, help="AWS region of the DynamoDB table") subparsers = parser.add_subparsers(dest="action", required=True, help="Action to perform") # compress sub-command compress_parser = subparsers.add_parser( "compress", help="Compress uncompressed JSON to gzip-compressed binary.", ) add_key_arguments(compress_parser) compress_parser.add_argument( "--execute", action="store_true", default=False, help="Actually perform the compression. Dry-run by default.", ) compress_parser.add_argument( "--workers", type=int, default=8, help="Number of concurrent workers (default: 8). Increase for faster throughput; decrease if throttling is excessive.", ) compress_parser.add_argument( "--failed-keys-output", required=False, help="Output file to write keys that failed compression. One key per line.", ) # delete-pickles sub-command delete_parser = subparsers.add_parser( "delete-pickles", help="Remove pickle data from items that have compressed JSON.", ) add_key_arguments(delete_parser) delete_parser.add_argument( "--execute", action="store_true", default=False, help="Actually perform the deletion. Dry-run by default.", ) delete_parser.add_argument( "--i-hereby-declare-we-no-longer-need-pickles", action="store_true", default=False, help="Required safety flag when using --execute. Confirms you understand this is destructive and irreversible.", ) delete_parser.add_argument( "--failed-keys-output", required=False, help="Output file to write keys that failed deletion. One key per line.", ) # status sub-command subparsers.add_parser( "status", help="Report the state of all keys in the table.", ) args = parser.parse_args() source_table = get_dynamodb_table(args.aws_profile, args.table_name, args.table_region) client = get_dynamodb_client(args.aws_profile, args.table_region) if args.action == "status": cmd_status(args, source_table) elif args.action == "compress": keys = resolve_keys(args, parser, source_table) cmd_compress(args, source_table, client, args.table_name, keys) elif args.action == "delete-pickles": keys = resolve_keys(args, parser, source_table) cmd_delete_pickles(args, source_table, keys) if __name__ == "__main__": main() ================================================ FILE: tools/inspect_serialized_state.py ================================================ """Read a state file or db and create a report which summarizes it's contents. Displays: State configuration Count of jobs Table of Jobs with start date of last run """ import optparse from tron.config import manager from tron.serialize.runstate import statemanager from tron.utils import chdir def parse_options(): parser = optparse.OptionParser() parser.add_option("-c", "--config-path", help="Path to the configuration.") parser.add_option( "-w", "--working-dir", default=".", help="Working directory to resolve relative paths.", ) opts, _ = parser.parse_args() if not opts.config_path: parser.error("A --config-path is required.") return opts def get_container(config_path): config_manager = manager.ConfigManager(config_path) return config_manager.load() def get_state(container): config = container.get_master().state_persistence state_manager = statemanager.PersistenceManagerFactory.from_config(config) names = container.get_job_names() return state_manager.restore(*names) def format_date(date_string): return date_string.strftime("%Y-%m-%d %H:%M:%S") if date_string else None def format_jobs(job_states): format = "%-30s %-8s %-5s %s\n" header = format % ("Name", "Enabled", "Runs", "Last Update") def max_run(item): start_time = filter(None, (run["start_time"] for run in item)) return max(start_time) if start_time else None def build(name, job): start_times = (max_run(job_run["runs"]) for job_run in job["runs"]) start_times = filter(None, start_times) last_run = format_date(max(start_times)) if start_times else None return format % (name, job["enabled"], len(job["runs"]), last_run) seq = sorted(build(*item) for item in job_states.items()) return header + "".join(seq) def display_report(state_config, job_states): print("State Config: %s" % str(state_config)) print("Total Jobs: %s" % len(job_states)) print("\n%s" % format_jobs(job_states)) def main(config_path, working_dir): container = get_container(config_path) config = container.get_master().state_persistence with chdir(working_dir): display_report(config, *get_state(container)) if __name__ == "__main__": opts = parse_options() main(opts.config_path, opts.working_dir) ================================================ FILE: tools/migration/migrate_config_0.2_to_0.3.py ================================================ """ Convert a 0.2.x Tron configuration file to the 0.3 format. Removes YAML anchors, references, and tags. Display warnings for NodePools under the nodes section. Display warnings for action requires sections that are not lists. """ import optparse import re import sys from tron import yaml YAML_TAG_RE = re.compile(r"!\w+\b") class Loader(yaml.Loader): """A YAML loader that does not clear its anchor mapping.""" def compose_document(self): self.get_event() node = self.compose_node(None, None) self.get_event() return node def strip_tags(source): """Remove YAML tags.""" return YAML_TAG_RE.sub("", source) def name_from_doc(doc): """Find the string identifier for a doc.""" if "name" in doc: return doc["name"] # Special case for node without a name, their name defaults to their hostname if set(doc.keys()) == {"hostname"}: return doc["hostname"] if set(doc.keys()) == {"nodes"}: raise ValueError("Please create a name for NodePool %s" % doc) raise ValueError("Could not find a name for %s" % doc) def warn_node_pools(content): doc = yaml.safe_load(content) node_pools = [node_doc for node_doc in doc["nodes"] if "nodes" in node_doc] if not node_pools: return print( "\n\nNode Pools should be moved into a node_pools section." + " The following node pools were found:\n" + "\n".join(str(n) for n in node_pools), file=sys.stderr, ) def warn_requires_list(content): action_names = [] doc = yaml.safe_load(content) for job in doc["jobs"]: for action in job["actions"]: if "requires" not in action: continue if isinstance(action["requires"], list): continue action_names.append("{}.{}".format(job["name"], action["name"])) if not action_names: return print( "\n\nAction requires should be a list." + " The following actions have requires that are not lists:\n" + "\n".join(action_names), file=sys.stderr, ) def create_loader(content): """Create a loader, and have it create the document from content.""" loader = Loader(content) loader.get_single_node() return loader def build_anchor_mapping(content): """Return a map of anchors to the new name to use.""" loader = create_loader(content) return { anchor_name: name_from_doc(loader.construct_document(yaml_node)) for anchor_name, yaml_node in loader.anchors.items() } def update_references(content): anchor_mapping = build_anchor_mapping(content) def key_length_func(kv): return len(kv[0]) anchors_by_length = sorted( anchor_mapping.items(), key=key_length_func, reverse=True, ) for anchor_name, string_name in anchors_by_length: # Remove the anchors content = re.sub(r"\s*&%s ?" % anchor_name, "", content) # Update the reference to use the string identifier content = re.sub( r"\*%s\b" % anchor_name, '"%s"' % string_name, content, ) return content def convert(source, dest): with open(source) as fh: content = fh.read() try: content = strip_tags(content) content = update_references(content) warn_node_pools(content) warn_requires_list(content) except yaml.scanner.ScannerError as e: print(f"Bad content: {e}\n{content}") with open(dest, "w") as fh: fh.write(content) if __name__ == "__main__": opt_parser = optparse.OptionParser() opt_parser.add_option("-s", dest="source", help="Source config filename.") opt_parser.add_option("-d", dest="dest", help="Destination filename.") opts, args = opt_parser.parse_args() if not opts.source or not opts.dest: print("Source and destination filenames required.", file=sys.stderr) sys.exit(1) convert(opts.source, opts.dest) ================================================ FILE: tools/migration/migrate_config_0.5.1_to_0.5.2.py ================================================ """Migrate a single configuration file (tron 0.5.1) to the new 0.5.2 multi-file format. Usage: python tools/migration/migrate_config_0.5.1_to_0.5.2.py \ --source old_config_filename \ --dest new_config_dir """ import optparse import os from tron.config import manager def parse_options(): parser = optparse.OptionParser() parser.add_option("-s", "--source", help="Path to old configuration file.") parser.add_option( "-d", "--dest", help="Path to new configuration directory.", ) opts, _ = parser.parse_args() if not opts.source: parser.error("--source is required") if not opts.dest: parser.error("--dest is required") return opts def main(source, dest): dest = os.path.abspath(dest) if not os.path.isfile(source): raise SystemExit("Error: Source (%s) is not a file" % source) if os.path.exists(dest): raise SystemExit("Error: Destination path (%s) already exists" % dest) old_config = manager.read_raw(source) manager.create_new_config(dest, old_config) if __name__ == "__main__": opts = parse_options() main(opts.source, opts.dest) ================================================ FILE: tools/migration/migrate_state.py ================================================ """ Migrate a state file/database from one StateStore implementation to another. It may also be used to add namespace names to jobs when upgrading from pre-0.5.2 to version 0.5.2. Usage: python tools/migration/migrate_state.py \ -s -d [ --namespace ] old_config.yaml and new_config.yaml should be configuration files with valid state_persistence sections. The state_persistence section configures the StateStore. Pre 0.5 state files can be read by the YamlStateStore. See the configuration documentation for more details on how to create state_persistence sections. """ import optparse from tron.config import manager from tron.config import schema from tron.serialize import runstate from tron.serialize.runstate.statemanager import PersistenceManagerFactory from tron.utils import chdir def parse_options(): parser = optparse.OptionParser() parser.add_option( "-s", "--source", help="The source configuration path which contains a state_persistence " "section configured for the state file/database.", ) parser.add_option( "-d", "--dest", help="The destination configuration path which contains a " "state_persistence section configured for the state file/database.", ) parser.add_option( "--source-working-dir", help="The working directory for source dir to resolve relative paths.", ) parser.add_option( "--dest-working-dir", help="The working directory for dest dir to resolve relative paths.", ) parser.add_option( "--namespace", action="store_true", help="Move jobs which are missing a namespace to the MASTER", ) opts, args = parser.parse_args() if not opts.source: parser.error("--source is required") if not opts.dest: parser.error("--dest is required.") return opts, args def get_state_manager_from_config(config_path, working_dir): """Return a state manager from the configuration.""" config_manager = manager.ConfigManager(config_path) config_container = config_manager.load() state_config = config_container.get_master().state_persistence with chdir(working_dir): return PersistenceManagerFactory.from_config(state_config) def get_current_config(config_path): config_manager = manager.ConfigManager(config_path) return config_manager.load() def add_namespaces(state_data): return {f"{schema.MASTER_NAMESPACE}.{name}": data for (name, data) in state_data.items()} def strip_namespace(names): return [name.split(".", 1)[1] for name in names] def convert_state(opts): source_manager = get_state_manager_from_config( opts.source, opts.source_working_dir, ) dest_manager = get_state_manager_from_config( opts.dest, opts.dest_working_dir, ) container = get_current_config(opts.source) msg = "Migrating state from %s to %s" print(msg % (source_manager._impl, dest_manager._impl)) job_names = container.get_job_names() if opts.namespace: job_names = strip_namespace(job_names) job_states = source_manager.restore( job_names, ) source_manager.cleanup() if opts.namespace: job_states = add_namespaces(job_states) for name, job in job_states.items(): dest_manager.save(runstate.JOB_STATE, name, job) print("Migrated %s jobs." % len(job_states)) dest_manager.cleanup() if __name__ == "__main__": opts, _args = parse_options() convert_state(opts) ================================================ FILE: tools/migration/migrate_state_1.3.15_to_1.4.0.py ================================================ import argparse import logging from tron.config import manager from tron.serialize import runstate from tron.serialize.runstate.statemanager import PersistenceManagerFactory from tron.utils import chdir def parse_args(): parser = argparse.ArgumentParser() parser.add_argument( "--back", help="Flag to migrate back from new state back to old state", action="store_true", default=False, ) parser.add_argument( "--working-dir", help="Working directory for the Tron daemon", required=True, ) parser.add_argument( "--config-path", help="Path in working dir with configs", required=True, ) return parser.parse_args() def create_job_runs_for_job(state_manager, job_name, job_state): for run in job_state["runs"]: run_num = run["run_num"] state_manager.save(runstate.JOB_RUN_STATE, f"{job_name}.{run_num}", run) run_nums = [run["run_num"] for run in job_state["runs"]] job_state["run_nums"] = run_nums # Note: not removing 'runs' from job_state for safety. # If Tron starts up correctly after the state migration, it will update the job state # and remove 'runs'. state_manager.save(runstate.JOB_STATE, job_name, job_state) def move_job_runs_to_job(state_manager, job_name, job_state): runs = state_manager._restore_runs_for_job(job_name, job_state) job_state["runs"] = runs state_manager.save(runstate.JOB_STATE, job_name, job_state) for run in runs: state_manager.delete(runstate.JOB_RUN_STATE, f'{job_name}.{run["run_num"]}') def update_state(state_manager, job_names, back): jobs = state_manager._restore_dicts(runstate.JOB_STATE, job_names) for job_name, job_state in jobs.items(): if back: move_job_runs_to_job(state_manager, job_name, job_state) else: create_job_runs_for_job(state_manager, job_name, job_state) def migrate_state(config_path, working_dir, back): with chdir(working_dir): config_manager = manager.ConfigManager(config_path) config_container = config_manager.load() job_names = config_container.get_job_names() state_config = config_container.get_master().state_persistence state_manager = PersistenceManagerFactory.from_config(state_config) update_state(state_manager, job_names, back) state_manager.cleanup() if __name__ == "__main__": # INFO for boto, DEBUG for all tron-related state logs logging.basicConfig(level=logging.INFO) logging.getLogger("tron").setLevel(logging.DEBUG) args = parse_args() migrate_state(args.config_path, args.working_dir, args.back) ================================================ FILE: tools/pickles_to_json.py ================================================ import argparse import math import os import pickle import boto3 import requests from boto3.resources.base import ServiceResource from tron.core.job import Job from tron.core.jobrun import JobRun from tron.serialize import runstate # Max DynamoDB object size is 400KB. Since we save two copies of the object (pickled and JSON), # we need to consider this max size applies to the entire item, so we use a max size of 200KB # for each version. OBJECT_SIZE = 200_000 def get_dynamodb_table( aws_profile: str | None = None, table: str = "infrastage-tron-state", region: str = "us-west-1" ) -> ServiceResource: """ Get the DynamoDB table resource. :param aws_profile: The name of the AWS profile to use. :param table: The name of the table to get. :param region: The region of the table. :return: The DynamoDB table resource. """ session = boto3.Session(profile_name=aws_profile) if aws_profile else boto3.Session() return session.resource("dynamodb", region_name=region).Table(table) def get_all_jobs(source_table: ServiceResource) -> list[str]: """ Scan the DynamoDB table and return a list of unique job keys. :param source_table: The DynamoDB table resource to scan. :return: A list of all job keys. """ items = scan_table(source_table) unique_keys = {item.get("key", "Unknown Key") for item in items} return list(unique_keys) def get_job_names(base_url: str) -> list[str]: """ Get the list of job names from the Tron API. :param base_url: The base URL of the Tron API. :return: A list of job names. """ try: full_url = f"http://{base_url}.yelpcorp.com:8089/api/jobs?include_job_runs=0" response = requests.get(full_url) response.raise_for_status() data = response.json() job_names = [job["name"] for job in data.get("jobs", [])] return job_names except requests.exceptions.RequestException as e: print(f"An error occurred: {e}") return [] def combine_pickle_partitions(source_table: ServiceResource, key: str) -> bytes: """ Load and combine all partitions of a pickled item from DynamoDB. :param source_table: The DynamoDB table resource. :param key: The primary key of the item to retrieve. :return: The combined pickled data as bytes. """ response = source_table.get_item(Key={"key": key, "index": 0}, ConsistentRead=True) if "Item" not in response: raise Exception(f"No item found for key {key} at index 0") item = response["Item"] num_partitions = int(item.get("num_partitions", 1)) combined_data = bytearray() for index in range(num_partitions): response = source_table.get_item(Key={"key": key, "index": index}, ConsistentRead=True) if "Item" not in response: raise Exception(f"Missing partition {index} for key {key}") item = response["Item"] combined_data.extend(item["val"].value) return bytes(combined_data) def dump_pickle_key(source_table: ServiceResource, key: str) -> None: """ Load the pickled data from DynamoDB for a given key, handling partitioned items, and print the full pickle data. :param source_table: The DynamoDB table resource. :param key: The primary key of the item to retrieve. """ try: pickled_data = combine_pickle_partitions(source_table, key) loaded_pickle = pickle.loads(pickled_data) print(f"Key: {key} - Pickle data:") print(loaded_pickle) except Exception as e: print(f"Key: {key} - Failed to load pickle: {e}") raise def dump_pickle_keys(source_table: ServiceResource, keys: list[str]) -> None: """ Load and print pickles for the given list of keys. :param source_table: The DynamoDB table resource. :param keys: A list of keys for which to load and print pickles. """ for key in keys: dump_pickle_key(source_table, key) def dump_json_key(source_table: ServiceResource, key: str) -> None: """ Load the JSON data from DynamoDB for a given key and print it. :param source_table: The DynamoDB table resource. :param key: The primary key of the item to retrieve. """ try: json_data = combine_json_partitions(source_table, key) if json_data is not None: print(f"Key: {key} - JSON data:") print(json_data) else: print(f"Key: {key} - No JSON value found") except Exception as e: print(f"Key: {key} - Failed to load JSON: {e}") def dump_json_keys(source_table: ServiceResource, keys: list[str]) -> None: """ Load and print JSON data for the given list of keys. :param source_table: The DynamoDB table resource. :param keys: A list of keys for which to load and print JSON data. """ for key in keys: dump_json_key(source_table, key) # TODO: clean up old run history for valid jobs? something something look at job_state, then whitelist those runs instead of whitelisting entire jobs def delete_keys(source_table: ServiceResource, keys: list[str]) -> None: """ Delete items with the given list of keys from the DynamoDB table. :param source_table: The DynamoDB table resource. :param keys: A list of keys to delete. """ total_keys = len(keys) deleted_count = 0 failure_count = 0 for key in keys: try: num_partitions = get_num_partitions(source_table, key) for index in range(num_partitions): source_table.delete_item(Key={"key": key, "index": index}) print(f"Key: {key} - Successfully deleted") deleted_count += 1 except Exception as e: print(f"Key: {key} - Failed to delete: {e}") failure_count += 1 print(f"Total keys: {total_keys}") print(f"Successfully deleted: {deleted_count}") print(f"Failures: {failure_count}") def get_num_partitions(source_table: ServiceResource, key: str) -> int: """ Get the number of partitions for a given key in the DynamoDB table. :param source_table: The DynamoDB table resource. :param key: The primary key of the item to retrieve. :return: The number of partitions for the key. """ response = source_table.get_item(Key={"key": key, "index": 0}, ConsistentRead=True) if "Item" not in response: return 0 item = response["Item"] num_partitions = int(item.get("num_partitions", 1)) num_json_val_partitions = int(item.get("num_json_val_partitions", 0)) return max(num_partitions, num_json_val_partitions) def combine_json_partitions(source_table: ServiceResource, key: str) -> str | None: """ Combine all partitions of a JSON item from DynamoDB. :param source_table: The DynamoDB table resource. :param key: The primary key of the item to retrieve. :return: The combined JSON data as a string, or None if not found. """ response = source_table.get_item(Key={"key": key, "index": 0}, ConsistentRead=True) if "Item" not in response: return None item = response["Item"] num_json_partitions = int(item.get("num_json_val_partitions", 0)) if num_json_partitions == 0: return None combined_json = "" for index in range(num_json_partitions): response = source_table.get_item(Key={"key": key, "index": index}, ConsistentRead=True) if "Item" not in response: raise Exception(f"Missing JSON partition {index} for key {key}") item = response["Item"] if "json_val" in item: combined_json += item["json_val"] else: raise Exception(f"No 'json_val' in partition {index} for key {key}") return combined_json def convert_pickle_to_json_and_update_table(source_table: ServiceResource, key: str, dry_run: bool = True) -> bool: """ Convert a single pickled item to JSON and update the DynamoDB entry. Returns True if the conversion was successful, False if skipped. Raises an exception if conversion fails. """ try: # Skip conversion for job_state MASTER and job_run_state MASTER jobs that are from infrastage testing (i.e., not real jobs) if key.startswith("job_state MASTER") or key.startswith("job_run_state MASTER"): print(f"Skipping conversion for key: {key}") return False pickled_data = combine_pickle_partitions(source_table, key) state_data = pickle.loads(pickled_data) state_type = key.split()[0] if state_type == runstate.JOB_STATE: json_data = Job.to_json(state_data) elif state_type == runstate.JOB_RUN_STATE: json_data = JobRun.to_json(state_data) else: # This will skip the state metadata and any other non-standard keys we have in the table print(f"Key: {key} - Unknown state type: {state_type}. Skipping.") return False num_json_partitions = math.ceil(len(json_data) / OBJECT_SIZE) for partition_index in range(num_json_partitions): json_partition = json_data[ partition_index * OBJECT_SIZE : min((partition_index + 1) * OBJECT_SIZE, len(json_data)) ] if not dry_run: source_table.update_item( Key={"key": key, "index": partition_index}, UpdateExpression="SET json_val = :json, num_json_val_partitions = :num_partitions", ExpressionAttributeValues={ ":json": json_partition, ":num_partitions": num_json_partitions, }, ) if dry_run: print(f"DRY RUN: Key: {key} - Pickle would have been converted to JSON and updated") else: print(f"Key: {key} - Pickle converted to JSON and updated") return True except Exception as e: print(f"Key: {key} - Failed to convert pickle to JSON: {e}") raise def convert_pickles_to_json_and_update_table( source_table: ServiceResource, keys: list[str], dry_run: bool = True, deprecated_keys_output: str | None = None, failed_keys_output: str | None = None, job_names: list[str] = [], ) -> None: """ Convert pickled items in the DynamoDB table to JSON and update the entries. :param source_table: The DynamoDB table resource. :param keys: List of keys to convert. :param dry_run: If True, simulate the conversion without updating the table. :param deprecated_keys_output: Output file to write deprecated keys to. :param failed_keys_output: Output file to write keys that failed to convert to. :param job_names: List of job names to use for filtering keys. """ total_keys = len(keys) converted_keys = 0 skipped_keys = 0 failed_keys = [] delete_keys = [] for key in keys: # Extract the job name from the key parts = key.split() if len(parts) < 2: continue state_type, job_info = parts[0], parts[1] # Ignore run_num for job_run_state keys if state_type == "job_run_state": job_name = ".".join(job_info.split(".")[:-1]) else: job_name = job_info if job_name not in job_names: delete_keys.append(key) continue try: result = convert_pickle_to_json_and_update_table(source_table, key, dry_run) if result: converted_keys += 1 else: skipped_keys += 1 except Exception as e: print(f"Key: {key} - Failed to convert pickle to JSON: {e}") failed_keys.append(key) print(f"Total keys processed: {total_keys}") print(f"Conversions attempted: {total_keys - skipped_keys}") print(f"Conversions succeeded: {converted_keys}") print(f"Conversions skipped: {skipped_keys}") print(f"Conversions failed: {len(failed_keys)}") print(f"Keys to be deleted: {len(delete_keys)}") if deprecated_keys_output: with open(deprecated_keys_output, "w") as f: for key in delete_keys: f.write(f"{key}\n") print(f"Deprecated keys have been written to {deprecated_keys_output}") if failed_keys_output: with open(failed_keys_output, "w") as f: for key in failed_keys: f.write(f"{key}\n") print(f"Failed have been written to {failed_keys_output}") if dry_run: print("Dry run complete. No changes were made to the DynamoDB table.") def scan_table(source_table: ServiceResource) -> list[dict]: """ Scan the DynamoDB table and return all items, handling pagination. :param source_table: The DynamoDB table resource to scan. :return: A list of all items in the table. """ items = [] response = source_table.scan() items.extend(response.get("Items", [])) while "LastEvaluatedKey" in response: response = source_table.scan(ExclusiveStartKey=response["LastEvaluatedKey"]) items.extend(response.get("Items", [])) return items def main(): parser = argparse.ArgumentParser( description="Utilities for working with pickles and JSON items in Tron's DynamoDB state store.", epilog=""" Actions: convert Convert pickled state data to JSON format and update the DynamoDB table. dump-pickle Load and print the pickles for specified keys. dump-json Load and print JSON data for specified keys. delete Delete the specified keys from the DynamoDB table. Examples: Validate pickles (write deprecated keys to deprecated_keys.txt, dry run by default): pickles_to_json.py --table-name infrastage-tron-state --table-region us-west-1 convert --all --deprecated-keys-output deprecated_keys.txt --tron-api tron-infrastage Convert all pickles to JSON (actually execute the update): pickles_to_json.py --table-name infrastage-tron-state --table-region us-west-1 convert --all --execute --tron-api tron-infrastage Convert specific pickles to JSON using keys from an input file: pickles_to_json.py --table-name infrastage-tron-state --table-region us-west-1 convert --keys-file input_keys.txt --execute --tron-api tron-infrastage Load and print specific JSON keys using keys from an input file: pickles_to_json.py --table-name infrastage-tron-state --table-region us-west-1 dump-json --keys-file input_keys.txt Delete specific keys (dry run by default): pickles_to_json.py --table-name infrastage-tron-state --table-region us-west-1 delete --keys "key1" "key2" Delete keys from an input file (actually execute the deletion): pickles_to_json.py --table-name infrastage-tron-state --table-region us-west-1 delete --keys-file input_keys.txt --execute """, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--aws-profile", default=os.environ.get("AWS_PROFILE", None), help="AWS profile to use (default: taken from AWS_PROFILE environment variable)", ) parser.add_argument("--table-name", required=True, help="Name of the DynamoDB table") parser.add_argument("--table-region", required=True, help="AWS region of the DynamoDB table") subparsers = parser.add_subparsers(dest="action", required=True, help="Action to perform") convert_parser = subparsers.add_parser( "convert", help="Convert pickled state data to JSON format and update the DynamoDB table." ) convert_parser.add_argument( "--tron-api", required=True, help="Base URL of the Tron API to fetch job names from.", ) convert_parser.add_argument( "--keys", nargs="+", required=False, help="Specific key(s) to perform the action on.", ) convert_parser.add_argument( "--keys-file", required=False, help="Input file containing keys to perform the action on. One key per line.", ) convert_parser.add_argument( "--deprecated-keys-output", required=False, help="Output file to write deprecated keys to. These are keys associated with jobs not present in Tron. One key per line.", ) convert_parser.add_argument( "--failed-keys-output", required=False, help="Output file to write keys that failed to convert. One key per line.", ) convert_parser.add_argument( "--all", action="store_true", help="Apply the action to all keys in the table.", ) convert_parser.add_argument( "--execute", action="store_true", default=False, help="Actually perform the conversion and update the DynamoDB table.", ) dump_pickle_parser = subparsers.add_parser("dump-pickle", help="Load and print the pickles for specified keys.") dump_pickle_parser.add_argument( "--keys", nargs="+", required=False, help="Specific key(s) to perform the action on.", ) dump_pickle_parser.add_argument( "--keys-file", required=False, help="Input file containing keys to perform the action on. One key per line.", ) dump_pickle_parser.add_argument( "--all", action="store_true", help="Apply the action to all keys in the table.", ) dump_json_parser = subparsers.add_parser("dump-json", help="Load and print JSON data for specified keys.") dump_json_parser.add_argument( "--keys", nargs="+", required=False, help="Specific key(s) to perform the action on.", ) dump_json_parser.add_argument( "--keys-file", required=False, help="Input file containing keys to perform the action on. One key per line.", ) dump_json_parser.add_argument( "--all", action="store_true", help="Apply the action to all keys in the table.", ) delete_keys_parser = subparsers.add_parser("delete", help="Delete the specified keys from the DynamoDB table.") delete_keys_parser.add_argument( "--keys", nargs="+", required=False, help="Specific key(s) to perform the action on.", ) delete_keys_parser.add_argument( "--keys-file", required=False, help="Input file containing keys to perform the action on. One key per line.", ) delete_keys_parser.add_argument( "--all", action="store_true", help="Apply the action to all keys in the table.", ) delete_keys_parser.add_argument( "--execute", action="store_true", default=False, help="Actually perform the deletion on the DynamoDB table.", ) args = parser.parse_args() source_table = get_dynamodb_table(args.aws_profile, args.table_name, args.table_region) if not args.keys and not args.keys_file and not args.all: parser.error("You must provide either --keys, --keys-file, or --all.") if args.all: print("Processing all keys in the table...") keys = get_all_jobs(source_table) else: keys = [] if args.keys: keys.extend(args.keys) if args.keys_file: try: with open(args.keys_file) as f: keys_from_file = [line.strip() for line in f if line.strip()] keys.extend(keys_from_file) except Exception as e: parser.error(f"Error reading keys from file {args.keys_file}: {e}") if not keys: parser.error("No keys provided. Please provide keys via --keys or --keys-file.") keys = list(set(keys)) if args.action == "convert": job_names = get_job_names(args.tron_api) convert_pickles_to_json_and_update_table( source_table, keys=keys, dry_run=not args.execute, deprecated_keys_output=args.deprecated_keys_output, failed_keys_output=args.failed_keys_output, job_names=job_names, ) elif args.action == "dump-pickle": dump_pickle_keys(source_table, keys) elif args.action == "dump-json": dump_json_keys(source_table, keys) elif args.action == "delete": if not args.execute: print(f"DRY RUN: Would delete {len(keys)} keys from the table '{args.table_name}'.") for key in keys: print(f"Would delete key: {key}") print("Dry run complete. No changes were made to the DynamoDB table.") else: confirm = ( input(f"Are you sure you want to delete {len(keys)} keys from the table '{args.table_name}'? [y/N]: ") .strip() .lower() ) if confirm in ("y", "yes"): delete_keys(source_table, keys) else: print("Deletion cancelled.") else: print(f"Unknown action: {args.action}") if __name__ == "__main__": main() ================================================ FILE: tools/sync_tron_state_from_k8s.py ================================================ """ Update tron state from k8s api if tron has not yet updated correctly Usage: python tools/sync_tron_state_from_k8s.py -c (--do-work|--num-runs N|--tronctl-wrapper tronctl-pnw-devc) This will search for completed pods in the cluster specified in the kubeconfig in the `tron` namespace and use tronctl to transition any whose states do not match. """ import argparse import base64 import hashlib import logging import os import subprocess import sys from typing import Any from kubernetes.client import V1Pod from task_processing.plugins.kubernetes.kube_client import KubeClient from tron.commands.client import Client from tron.commands.cmd_utils import get_client_config POD_STATUS_TO_TRON_STATE = { "Succeeded": "success", "Failed": "fail", "Unknown": "Unknown", # This should never really happen } TRON_MODIFIABLE_STATES = [ "starting", # stuck jobs "running", # stuck jobs "unknown", "lost", ] log = logging.getLogger("sync_tron_from_k8s") # NOTE: Copied from paasta_tools.kubernetes_tools, if it changes there it must be updated here def limit_size_with_hash(name: str, limit: int = 63, suffix: int = 4) -> str: """Returns `name` unchanged if it's length does not exceed the `limit`. Otherwise, returns truncated `name` with it's hash of size `suffix` appended. base32 encoding is chosen as it satisfies the common requirement in various k8s names to be alphanumeric. """ if len(name) > limit: digest = hashlib.md5(name.encode()).digest() hash = base64.b32encode(digest).decode().replace("=", "").lower() return f"{name[:(limit-suffix-1)]}-{hash[:suffix]}" else: return name def parse_args(): parser = argparse.ArgumentParser() parser.add_argument( "--kubeconfig-path", dest="kubeconfig_path", help="KUBECONFIG path; multiple can be specified to find pods in multiple clusters", nargs="+", ) parser.add_argument( "--kubecontext", dest="kubecontext", help="kubecontext to use from specified kubeconfig. multiple can be specified to find pods in multiple clusters, ONLY if a single kubeconfig-path is provided", nargs="*", ) parser.add_argument( "--do-work", dest="do_work", action="store_true", default=False, help="Actually modify tron actions that need updating; without this flag we will only print those that would be updated", ) parser.add_argument("--tron-url", default=None, help="Tron url (default will read from paasta tron config)") parser.add_argument( "--tronctl-wrapper", default="tronctl", dest="tronctl_wrapper", help="Tronctl wrapper to use (will not use wrapper by default)", ) parser.add_argument("-n", "--num-runs", dest="num_runs", default=100, help="Maximum number of job runs to retrieve") parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=False, help="Verbose logging") args = parser.parse_args() # We can only have multiple kubeconfigs, or multiple contexts with a single config if len(args.kubeconfig_path) > 1 and args.kubecontext: parser.error("You can only specify a single --kubeconfig-path if specifying --kubecontext arguments.") # tron's base level is critical, not info, adjust accoringly if args.verbose: level = logging.DEBUG tron_level = logging.WARN else: level = logging.INFO tron_level = logging.CRITICAL logging.basicConfig(level=level, stream=sys.stdout) tron_client_logger = logging.getLogger("tron.commands.client") tron_client_logger.setLevel(tron_level) # We also don't want kube_client debug logs kube_logger = logging.getLogger("kubernetes.client.rest") kube_logger.setLevel(logging.INFO) return args def fetch_pods(kubeconfig_path: str, kubecontext: str | None) -> dict[str, V1Pod]: if kubecontext: # KubeClient only uses the environment variable os.environ["KUBECONTEXT"] = kubecontext kube_client = KubeClient(kubeconfig_path=kubeconfig_path, user_agent="sync_tron_state_from_k8s") # Bit of a hack, no helper to fetch pods so reach into core api completed_pod_list = kube_client.core.list_namespaced_pod( namespace="tron", ) return {pod.metadata.name: pod for pod in completed_pod_list.items} def get_tron_state_from_api(tron_server: str, num_runs: int = 100) -> list[dict[str, dict[Any, Any]]]: if not tron_server: client_config = get_client_config() tron_server = client_config.get("server", "http://localhost:8089") client = Client(tron_server) # /jobs returns only the latest 5 runs, we'll need to request all runs instead ourselves jobs = client.jobs( include_job_runs=False, include_action_runs=False, include_action_graph=False, include_node_pool=False, ) for job in jobs: # Update job URL to be used with API instead of web url = f'/api{job["url"]}' log.debug(f'Fetching job {job["name"]} at {url}') job_runs = client.job( url, include_action_runs=True, count=num_runs, # TODO: fetch job run_limit and use that for count ? ) job["runs"] = job_runs["runs"] return jobs def get_matching_pod(action_run: dict[str, Any], pods: dict[str, V1Pod]) -> V1Pod | None: """Given a tron action_run, try to find the right pod that matches.""" action_name = action_run["action_name"] job_name = action_run["job_name"] run_num = action_run["run_num"] service, job = job_name.split(".") instance_name = f"{job}.{action_name}" sanitized_instance_name = limit_size_with_hash(instance_name) matching_pods = sorted( [ pod for pod in pods.values() if pod.metadata.labels["paasta.yelp.com/service"] == service and pod.metadata.labels["paasta.yelp.com/instance"] == sanitized_instance_name and pod.metadata.labels["tron.yelp.com/run_num"] == run_num ], # If action has retries, there will be multiple pods w/ same job_run; we only want the latest key=lambda pod: pod.metadata.creation_timestamp, reverse=True, ) return ( matching_pods[0] if matching_pods and matching_pods[0].status.phase in POD_STATUS_TO_TRON_STATE.keys() else None ) def get_desired_state_from_pod(pod: V1Pod) -> str: k8s_state = pod.status.phase return POD_STATUS_TO_TRON_STATE.get(k8s_state, "NoMatch") def update_tron_from_pods( jobs: list[dict[str, Any]], pods: dict[str, V1Pod], tronctl_wrapper: str = "tronctl", do_work: bool = False ): updated = [] error = [] for job in jobs: if job["runs"]: # job_runs for job_run in job["runs"]: # actions for this job_run for action in job_run.get("runs", []): action_run_id = action["id"] if action["state"] in TRON_MODIFIABLE_STATES: pod = get_matching_pod(action, pods) if pod: desired_state = get_desired_state_from_pod(pod) if action["state"] != desired_state: log.debug(f'{action_run_id} state {action["state"]} needs updating to {desired_state}') cmd = [tronctl_wrapper, desired_state, action_run_id] if do_work: # tronctl-$cluster success/fail svc.job.run.action try: log.info(f"Running {cmd}") proc = subprocess.run(cmd, capture_output=True, text=True) if proc.returncode != 0: log.error(f"Got non-zero exit code: {proc.returncode}") log.error(f"\t{proc.stderr}") error.append(action_run_id) updated.append(action_run_id) except Exception: log.exception("ERROR: Hit exception:") error.append(action_run_id) else: log.info(f"Dry-Run: Would run {cmd}") updated.append(action_run_id) else: log.debug(f"action run {action_run_id} not found in list of finished pods, no action taken") else: log.debug(f'Action state {action["state"]} for {action_run_id} not modifiable, no action taken') log.info(f"Updated {len(updated)} actions: {','.join(updated)}") log.info(f"Hit {len(error)} errors on actions: {','.join(error)}") return {"updated": updated, "error": error} if __name__ == "__main__": args = parse_args() jobs = get_tron_state_from_api(args.tron_url, args.num_runs) log.debug(f"Found {len(jobs)} jobs.") pods = {} kube_client_args = ( [(args.kubeconfig_path[0], kubecontext) for kubecontext in args.kubecontext] if args.kubecontext else [(kubeconfig_path, None) for kubeconfig_path in args.kubeconfig_path] ) for kubeconfig_path, kubecontext in kube_client_args: pods.update(fetch_pods(kubeconfig_path, kubecontext)) log.debug(f"Found {len(pods.keys())} pods.") update_tron_from_pods(jobs, pods, args.tronctl_wrapper, args.do_work) ================================================ FILE: tox.ini ================================================ [tox] envlist = py310 [testenv] basepython = python3.10 deps = --requirement={toxinidir}/requirements.txt --requirement={toxinidir}/requirements-dev.txt usedevelop = true passenv = USER PIP_INDEX_URL commands = pre-commit install -f --install-hooks pre-commit run --all-files # tron has been around for a while, so we'll need to slowly add types or make an effort # to get it mypy-clean in one shot - until then, let's only check files that we've added types to mypy --package tron check-requirements # optionally install yelpy requirements - this is after check-requirements since # check-requirements doesn't understand these extra requirements -pip install -r yelp_package/extra_requirements_yelp.txt # we then run tests at the very end so that we can run tests with yelpy requirements py.test -s {posargs:tests} [flake8] ignore = E501,E265,E241,E704,E251,W504,E231,W503,E203 [testenv:docs] deps = --requirement={toxinidir}/requirements-docs.txt --requirement={toxinidir}/requirements.txt whitelist_externals= mkdir commands= /bin/rm -rf docs/source/generated/ # The last arg to apidoc is a list of excluded paths sphinx-apidoc -f -e -o docs/source/generated/ tron mkdir -p docs sphinx-build -b html -d docs/_build docs/source docs/_build/html [testenv:itest] commands = make deb_jammy make _itest_jammy ================================================ FILE: tron/__init__.py ================================================ # Copyright 2015-2016 Yelp Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # It is imperative that this file not contain any imports from our # dependencies. Since this file is imported from setup.py in the # setup phase, the dependencies may not exist on disk yet. # # Don't bump version manually. See `make release` docs in ./Makefile __version__ = "3.10.0" ================================================ FILE: tron/actioncommand.py ================================================ import json import logging import os from io import StringIO from shlex import quote from typing import Any from tron.config import schema from tron.serialize import filehandler from tron.utils import timeutils from tron.utils.observer import Observable from tron.utils.persistable import Persistable from tron.utils.state import Machine log = logging.getLogger(__name__) class ActionCommand(Observable): """An ActionCommand encapsulates a runnable task that is passed to a node for execution. A Node calls: started (when the command starts) exited (when the command exits) write_ (when output is received) done (when the command is finished) """ PENDING = "pending" RUNNING = "running" EXITING = "exiting" COMPLETE = "complete" FAILSTART = "failstart" STATE_MACHINE = Machine( PENDING, **{ PENDING: { "start": RUNNING, "exit": FAILSTART, }, RUNNING: { "exit": EXITING, }, EXITING: { "close": COMPLETE, }, }, ) STDOUT = ".stdout" STDERR = ".stderr" def __init__(self, id, command, serializer=None): super().__init__() self.id = id self.command = command self.machine = Machine.from_machine(ActionCommand.STATE_MACHINE) self.exit_status = None self.start_time = None self.end_time = None if serializer: self.stdout = serializer.open(self.STDOUT) self.stderr = serializer.open(self.STDERR) else: self.stdout = filehandler.NullFileHandle self.stderr = filehandler.NullFileHandle @property def state(self): return self.machine.state def transition_and_notify(self, target): if self.machine.transition(target): self.notify(self.state) return True def started(self): if self.machine.check("start"): self.start_time = timeutils.current_timestamp() return self.transition_and_notify("start") def exited(self, exit_status): if self.machine.check("exit"): self.end_time = timeutils.current_timestamp() self.exit_status = exit_status return self.transition_and_notify("exit") def write_stderr(self, value): self.stderr.write(value) def write_stdout(self, value): self.stdout.write(value) def done(self): if self.machine.check("close"): self.stdout.close() self.stderr.close() return self.transition_and_notify("close") def handle_errback(self, result): """Handle an unexpected error while being run. This will likely be an interval error. Cleanup the state of this ActionCommand and log something useful for debugging. """ log.error(f"Unknown failure for {self}, {str(result)}") self.exited(result) self.done() @property def is_unknown(self): return self.exit_status is None @property def is_failed(self): return bool(self.exit_status) @property def is_complete(self): """Complete implies done and success.""" return self.machine.state == ActionCommand.COMPLETE @property def is_done(self): """Done implies no more work will be done, but might not be success.""" return self.machine.state in ( ActionCommand.COMPLETE, ActionCommand.FAILSTART, ) def __repr__(self): return f"ActionCommand {self.id} {self.command}: {self.state}" class StringBufferStore: """A serializer object which can be passed to ActionCommand as a serializer, but stores streams in memory. """ def __init__(self): self.buffers = {} def open(self, name): return self.buffers.setdefault(name, StringIO()) def clear(self): self.buffers.clear() # TODO: TRON-2304 - Cleanup NoActionRunnerFactory class NoActionRunnerFactory: """Action runner factory that does not wrap the action run command.""" @classmethod def create(cls, id, command, serializer): return ActionCommand(id, command, serializer) @classmethod def build_stop_action_command(cls, _id, _command): """It is not possible to stop action commands without a runner.""" raise NotImplementedError("An action_runner is required to stop.") @staticmethod def from_json(): return None @staticmethod def to_json(): return None class SubprocessActionRunnerFactory(Persistable): """Run actions by wrapping them in `action_runner.py`.""" runner_exec_name = "action_runner.py" status_exec_name = "action_status.py" def __init__(self, status_path, exec_path): self.status_path = status_path self.exec_path = exec_path @classmethod def from_config(cls, config): return cls(config.remote_status_path, config.remote_exec_path) def create(self, id, command, serializer): command = self.build_command(id, command, self.runner_exec_name) return ActionCommand(id, command, serializer) def build_command(self, id, command, exec_name): status_path = os.path.join(self.status_path, id) runner_path = os.path.join(self.exec_path, exec_name) return f"{quote(runner_path)} {quote(status_path)} {quote(command)} {quote(id)}" def build_stop_action_command(self, id, command): command = self.build_command(id, command, self.status_exec_name) run_id = f"{id}.{command}" return ActionCommand(run_id, command, StringBufferStore()) def __eq__(self, other): return ( self.__class__ == other.__class__ and self.status_path == other.status_path and self.exec_path == other.exec_path ) def __ne__(self, other): return not self == other @staticmethod def from_json(state_data: str) -> dict[str, Any]: try: json_data = json.loads(state_data) deserialized_data = { "status_path": json_data["status_path"], "exec_path": json_data["exec_path"], } return deserialized_data except Exception: log.exception("Error deserializing SubprocessActionRunnerFactory from JSON") raise @staticmethod def to_json(state_data: dict) -> str: try: return json.dumps( { "status_path": state_data["status_path"], "exec_path": state_data["exec_path"], } ) except KeyError: log.exception("Missing key in state_data:") raise except Exception: log.exception("Error serializing SubprocessActionRunnerFactory to JSON:") raise def create_action_runner_factory_from_config(config): """A factory-factory method which returns a callable that can be used to create ActionCommand objects. The factory definition should match the constructor for ActionCommand. """ if not config or config.runner_type == schema.ActionRunnerTypes.none.value: return NoActionRunnerFactory() elif config.runner_type == schema.ActionRunnerTypes.subprocess.value: return SubprocessActionRunnerFactory.from_config(config) else: raise ValueError("Unknown runner type: %s", config.runner_type) ================================================ FILE: tron/api/__init__.py ================================================ ================================================ FILE: tron/api/adapter.py ================================================ """ Classes which create external representations of core objects. This allows the core objects to remain decoupled from the API and clients. These classes act as an adapter between the data format api clients expect, and the internal data of an object. """ import functools import os.path import time from collections.abc import Callable from typing import Any from typing import TypeVar from urllib.parse import quote from tron import actioncommand from tron import scheduler from tron.core.actionrun import KubernetesActionRun from tron.serialize import filehandler from tron.utils import timeutils from tron.utils.logreader import read_log_stream_for_action_run from tron.utils.timeutils import delta_total_seconds R = TypeVar("R") class ReprAdapter: """Creates a dictionary from the given object for a set of rules.""" field_names: list[str] = [] translated_field_names: list[str] = [] def __init__(self, internal_obj): self._obj = internal_obj self.fields = self._get_field_names() self.translators = self._get_translation_mapping() def _get_field_names(self): return self.field_names def _get_translation_mapping(self): return {field_name: getattr(self, "get_%s" % field_name) for field_name in self.translated_field_names} def get_repr(self): repr_data = {field: getattr(self._obj, field) for field in self.fields} translated = {field: func() for field, func in self.translators.items()} repr_data.update(translated) return repr_data def adapt_many(adapter_class, seq, *args, **kwargs): return [adapter_class(item, *args, **kwargs).get_repr() for item in seq if item is not None] def toggle_flag( flag_name: str, ) -> Callable[ # the typing here is funky, this is a decorator factory that returns a decorator # which takes a function with "one" argument (really, that argument is just self) and returns some type R # where R is the return type of the decorated function [Callable[[Any], R]], # and that decorator returns another callable (since this is a decorator factory) that takes self (again, the Any) # and returns None (if the flag is False) or R (if True) Callable[[Any], R | None], ]: """Create a decorator which checks if flag_name is true before running the wrapped function. If False returns None. """ def wrap(f: Callable[[Any], R]) -> Callable[[Any], R | None]: @functools.wraps(f) def wrapper(self: Any) -> R | None: if getattr(self, flag_name): return f(self) return None return wrapper return wrap class RunAdapter(ReprAdapter): """Base class for JobRun and ActionRun adapters.""" def get_state(self): return self._obj.state def get_node(self): return NodeAdapter(self._obj.node).get_repr() def get_duration(self): duration = timeutils.duration(self._obj.start_time, self._obj.end_time) return str(duration or "") class ActionRunAdapter(RunAdapter): """Adapt a JobRun and an Action name to an external representation of an ActionRun. """ field_names = [ "id", "start_time", "end_time", "exit_status", "action_name", "exit_statuses", "retries_remaining", "original_command", ] translated_field_names = [ "state", "node", "command", "raw_command", "requirements", "meta", "stdout", "stderr", "duration", "job_name", "run_num", "retries_delay", "in_delay", "triggered_by", "trigger_downstreams", ] def __init__( self, action_run, job_run=None, max_lines=10, include_stdout=False, include_stderr=False, include_meta=False, ): super().__init__(action_run) self.job_run = job_run self.max_lines = max_lines or None self.include_stdout = include_stdout self.include_stderr = include_stderr self.include_meta = include_meta def get_raw_command(self): return self._obj.command_config.command def get_command(self): return self._obj.rendered_command @toggle_flag("job_run") def get_requirements(self): action_name = self._obj.action_name required = self.job_run.action_graph.get_dependencies(action_name) return [act.name for act in required] def _get_serializer(self, path: str | None = None) -> filehandler.OutputStreamSerializer: base_path = filehandler.OutputPath(path) if path else self._obj.output_path return filehandler.OutputStreamSerializer(base_path) def _get_alternate_output_paths(self): try: namespace, jobname, run_num, action = self._obj.id.split(".") except Exception: return None # Check to see if the output might have ended up in any alternate locations. for alt_path in self._obj.STDOUT_PATHS: formatted_alt_path = os.path.join( # This ugliness is getting the "root output directory" self._obj.context.next.next.base.job.output_path.base, alt_path.format( namespace=namespace, jobname=jobname, run_num=run_num, action=action, ), ) if os.path.exists(formatted_alt_path): yield formatted_alt_path @toggle_flag("include_meta") def get_meta(self) -> list[str]: if not isinstance(self._obj, KubernetesActionRun): return ["When this action is migrated to Kubernetes, this will contain Tron/task_processing output."] # We're reusing the "old" (i.e., SSH/Mesos) logging files for task_processing output since # that won't make it into anything but Splunk filename = actioncommand.ActionCommand.STDERR output: list[str] = self._get_serializer().tail(filename, self.max_lines) if not output: for alt_path in self._get_alternate_output_paths(): output = self._get_serializer(alt_path).tail(filename, self.max_lines) if output: return output return output @toggle_flag("include_stdout") def get_stdout(self) -> list[str]: if isinstance(self._obj, KubernetesActionRun): # it's possible that we have a job that logs to the samestream as another job on a # different master (e.g., 1 job in pnw-devc and another in norcal-devc), so we # additionally filter by the cluster in each log message. # we get this information from the last attempt for this ActionRun, but # all of the attempts should always have the same value. This value is guaranteed # to be here as it's part of the PaaSTA Contract, but there's also a fallback in # read_log_stream_for_action_run() to use the current superregion for the tron # master should something go horribly wrong paasta_cluster = None if self._obj.attempts: paasta_cluster = self._obj.attempts[-1].command_config.env.get("PAASTA_CLUSTER") return read_log_stream_for_action_run( action_run_id=self._obj.id, component="stdout", # we update the start time of an ActionRun on a retry so we can't just use # that start time to figure out when we should start displaying logs for. # instead, we use the first attempt's start time as the date from which to # start getting logs from and the last attempt's end time as the date at # which we stop getting logs from. # in the case of an action that completed on its initial run, there will # only be one attempt, but that's fine as these single attempts will still # have the correct information. # XXX: this is suboptimal if there's many days between retries min_date=self._obj.attempts[0].start_time if self._obj.attempts else None, max_date=self._obj.attempts[-1].end_time if self._obj.attempts else None, paasta_cluster=paasta_cluster, max_lines=self.max_lines, ) filename = actioncommand.ActionCommand.STDOUT output = self._get_serializer().tail(filename, self.max_lines) if not output: for alt_path in self._get_alternate_output_paths(): output = self._get_serializer(alt_path).tail(filename, self.max_lines) if output: break return output @toggle_flag("include_stderr") def get_stderr(self) -> list[str]: if isinstance(self._obj, KubernetesActionRun): # it's possible that we have a job that logs to the samestream as another job on a # different master (e.g., 1 job in pnw-devc and another in norcal-devc), so we # additionally filter by the cluster in each log message. # we get this information from the last attempt for this ActionRun, but # all of the attempts should always have the same value. This value is guaranteed # to be here as it's part of the PaaSTA Contract, but there's also a fallback in # read_log_stream_for_action_run() to use the current superregion for the tron # master should something go horribly wrong paasta_cluster = None if self._obj.attempts: paasta_cluster = self._obj.attempts[-1].command_config.env.get("PAASTA_CLUSTER") return read_log_stream_for_action_run( action_run_id=self._obj.id, component="stderr", # we update the start time of an ActionRun on a retry so we can't just use # that start time to figure out when we should start displaying logs for. # instead, we use the first attempt's start time as the date from which to # start getting logs from and the last attempt's end time as the date at # which we stop getting logs from. # in the case of an action that completed on its initial run, there will # only be one attempt, but that's fine as these single attempts will still # have the correct information. # XXX: this is suboptimal if there's many days between retries min_date=self._obj.attempts[0].start_time if self._obj.attempts else None, max_date=self._obj.attempts[-1].end_time if self._obj.attempts else None, paasta_cluster=paasta_cluster, max_lines=self.max_lines, ) filename = actioncommand.ActionCommand.STDERR output = self._get_serializer().tail(filename, self.max_lines) if not output: for alt_path in self._get_alternate_output_paths(): output = self._get_serializer(alt_path).tail(filename, self.max_lines) if output: break return output def get_job_name(self): return self._obj.job_run_id.rsplit(".", 1)[-2] def get_run_num(self): return self._obj.job_run_id.split(".")[-1] def get_retries_delay(self): if self._obj.retries_delay: return str(self._obj.retries_delay) def get_in_delay(self): if self._obj.in_delay is not None: return self._obj.in_delay.getTime() - time.time() def get_triggered_by(self) -> str: remaining = set(self._obj.remaining_triggers) all_triggers = sorted(self._obj.rendered_triggers) return ", ".join(f"{trig}{' (done)' if trig not in remaining else ''}" for trig in all_triggers) def get_trigger_downstreams(self) -> str: triggers_to_emit = self._obj.triggers_to_emit() return ", ".join(sorted(triggers_to_emit)) class ActionGraphAdapter: def __init__(self, action_graph): self.action_graph = action_graph def get_repr(self): def build(action_name): action = self.action_graph[action_name] dependencies = self.action_graph.get_dependencies(action_name, include_triggers=True) return { "name": action.name, "command": action.command, "dependencies": [d.name for d in dependencies], } return [build(action) for action in self.action_graph.names(include_triggers=True)] class ActionRunGraphAdapter: def __init__(self, action_run_collection): self.action_runs = action_run_collection def get_repr(self): def build(action_run): graph = self.action_runs.action_graph dependencies = graph.get_dependencies(action_run.action_name, include_triggers=True) return { "id": action_run.id, "name": action_run.action_name, "command": action_run.rendered_command, "raw_command": action_run.command_config.command, "state": action_run.state, "start_time": action_run.start_time, "end_time": action_run.end_time, "dependencies": [d.name for d in dependencies], } def build_trigger(trigger_name): graph = self.action_runs.action_graph trigger = graph[trigger_name] dependencies = graph.get_dependencies(trigger_name, include_triggers=True) return { "name": trigger.name, "command": trigger.command, "dependencies": [d.name for d in dependencies], "state": "unknown", # TODO: TRON-2382: why is this hardcoded and never updated? Can we update this after improving our API timings? } return [build(action_run) for action_run in self.action_runs] + [ build_trigger(trigger_name) for trigger_name in self.action_runs.action_graph.all_triggers ] class JobRunAdapter(RunAdapter): field_names = [ "id", "run_num", "run_time", "start_time", "end_time", "manual", "job_name", ] translated_field_names = [ "state", "node", "duration", "url", "runs", "action_graph", ] def __init__( self, job_run, include_action_runs=False, include_action_graph=False, ): super().__init__(job_run) self.include_action_runs = include_action_runs self.include_action_graph = include_action_graph def get_url(self): return f"/jobs/{self._obj.job_name}/{self._obj.run_num}" @toggle_flag("include_action_runs") def get_runs(self): return adapt_many(ActionRunAdapter, self._obj.action_runs, self._obj) @toggle_flag("include_action_graph") def get_action_graph(self): return ActionRunGraphAdapter(self._obj.action_runs).get_repr() class JobAdapter(ReprAdapter): field_names = ["status", "all_nodes", "allow_overlap", "queueing"] translated_field_names = [ "name", "scheduler", "action_names", "node_pool", "last_success", "next_run", "url", "runs", "max_runtime", "action_graph", "monitoring", "expected_runtime", "actions_expected_runtime", ] def __init__( self, job, include_job_runs=False, include_action_runs=False, include_action_graph=True, include_node_pool=True, num_runs=None, ): super().__init__(job) self.include_job_runs = include_job_runs self.include_action_runs = include_action_runs self.include_action_graph = include_action_graph self.include_node_pool = include_node_pool self.num_runs = num_runs def get_name(self): return self._obj.get_name() def get_monitoring(self): return self._obj.get_monitoring() def get_scheduler(self): return SchedulerAdapter(self._obj.scheduler).get_repr() def get_action_names(self): return list(self._obj.action_graph.names()) @toggle_flag("include_node_pool") def get_node_pool(self): return NodePoolAdapter(self._obj.node_pool).get_repr() def get_last_success(self): last_success = self._obj.runs.last_success return last_success.end_time if last_success else None def get_next_run(self): next_run = self._obj.runs.next_run return next_run.run_time if next_run else None def get_url(self): return f"/jobs/{quote(self._obj.get_name())}" @toggle_flag("include_job_runs") def get_runs(self): runs = adapt_many( JobRunAdapter, list(self._obj.runs)[: self.num_runs or None], self.include_action_runs, ) return runs def get_max_runtime(self): return str(self._obj.max_runtime) def get_expected_runtime(self): return delta_total_seconds(self._obj.expected_runtime) def get_actions_expected_runtime(self): return self._obj.action_graph.expected_runtime @toggle_flag("include_action_graph") def get_action_graph(self): return ActionGraphAdapter(self._obj.action_graph).get_repr() class JobIndexAdapter(ReprAdapter): translated_field_names = ["name", "actions"] def get_name(self): return self._obj.get_name() def get_actions(self): def adapt_run(run): return {"name": run.action_name, "command": run.command_config.command} job_run = self._obj.get_runs().get_newest() if not job_run: return [] return [adapt_run(action_run) for action_run in job_run.action_runs] class SchedulerAdapter(ReprAdapter): translated_field_names = ["value", "type", "jitter"] def get_value(self): return self._obj.get_value() def get_type(self): return self._obj.get_name() def get_jitter(self): return scheduler.get_jitter_str(self._obj.get_jitter()) class EventAdapter(ReprAdapter): field_names = ["name", "entity", "time"] translated_field_names = ["level"] def get_level(self): return self._obj.level.label class NodeAdapter(ReprAdapter): field_names = ["name", "hostname", "username", "port"] class NodePoolAdapter(ReprAdapter): translated_field_names = ["name", "nodes"] def get_name(self): return self._obj.get_name() def get_nodes(self): return adapt_many(NodeAdapter, self._obj.get_nodes()) ================================================ FILE: tron/api/async_resource.py ================================================ import threading import time from twisted.internet import threads from twisted.web import server from tron.metrics import timer def report_resource_request(resource, request, duration_ms): timer( name=f"tron.api.{resource.__class__.__name__}", delta=duration_ms, dimensions={"method": request.method.decode()}, ) class AsyncResource: capacity = 10 semaphore = threading.Semaphore(value=capacity) lock = threading.Lock() @staticmethod def finish(result, request, resource): result, duration_ms = result request.write(result) request.finish() report_resource_request(resource, request, duration_ms) @staticmethod def process(fn, resource, request): start = time.time() with AsyncResource.semaphore: result = fn(resource, request) duration_ms = 1000 * (time.time() - start) return result, duration_ms @staticmethod def bounded(fn): def wrapper(resource, request): d = threads.deferToThread( AsyncResource.process, fn, resource, request, ) d.addCallback(AsyncResource.finish, request, resource) d.addErrback(request.processingFailed) return server.NOT_DONE_YET return wrapper @staticmethod def exclusive(fn): def wrapper(resource, request): # ensures only one exclusive request starts consuming the semaphore start = time.time() with AsyncResource.lock: # this will wait until all bounded requests finished processing for _ in range(AsyncResource.capacity): AsyncResource.semaphore.acquire() try: return fn(resource, request) finally: for _ in range(AsyncResource.capacity): AsyncResource.semaphore.release() duration_ms = 1000 * (time.time() - start) report_resource_request(resource, request, duration_ms) return wrapper ================================================ FILE: tron/api/auth.py ================================================ import logging import os import re from functools import lru_cache from typing import NamedTuple import cachetools.func import requests from twisted.web.server import Request logger = logging.getLogger(__name__) AUTH_CACHE_SIZE = 50000 AUTH_CACHE_TTL = 30 * 60 SERVICE_NAME_PATH_PATTERN = re.compile(r"^/api/jobs/([^/.]+)") class AuthorizationOutcome(NamedTuple): authorized: bool reason: str class AuthorizationFilter: """API request authorization via external system""" def __init__(self, endpoint: str, enforce: bool): """Constructor :param str endpoint: HTTP endpoint of external authorization system :param bool enforce: whether to enforce authorization decisions """ self.endpoint = endpoint self.enforce = enforce self.session = requests.Session() @classmethod @lru_cache(maxsize=1) def get_from_env(cls) -> "AuthorizationFilter": return cls( endpoint=os.getenv("API_AUTH_ENDPOINT", ""), enforce=bool(os.getenv("API_AUTH_ENFORCE", "")), ) def is_request_authorized(self, request: Request) -> AuthorizationOutcome: """Check if API request is authorized :param Request request: API request object :return: auth outcome """ if not self.endpoint: return AuthorizationOutcome(True, "Auth not enabled") token = (request.getHeader("Authorization") or "").strip() token = token.split()[-1] if token else "" # removes "Bearer" prefix url_path = request.path.decode() if request.path is not None else "" # type: ignore[attr-defined] # mypy does not like what twisted is doing here auth_outcome = self._is_request_authorized_impl( # path and method are byte arrays in twisted path=url_path, token=token, method=request.method.decode(), service=self._extract_service_from_path(url_path), ) return auth_outcome if self.enforce else AuthorizationOutcome(True, "Auth dry-run") @cachetools.func.ttl_cache(maxsize=AUTH_CACHE_SIZE, ttl=AUTH_CACHE_TTL) def _is_request_authorized_impl( self, path: str, token: str, method: str, service: str | None, ) -> AuthorizationOutcome: """Check if API request is authorized :param str path: API path :param str token: authentication token :param str method: http method :return: auth outcome """ try: response = self.session.post( url=self.endpoint, json={ "input": { "path": path, "backend": "tron", "token": token, "method": method.lower(), "service": service, }, }, timeout=2, ).json() except Exception as e: logger.exception(f"Issue communicating with auth endpoint: {e}") return AuthorizationOutcome(False, "Auth backend error") auth_result_allowed = response.get("result", {}).get("allowed") if auth_result_allowed is None: return AuthorizationOutcome(False, "Malformed auth response") if not auth_result_allowed: reason = response["result"].get("reason", "Denied") return AuthorizationOutcome(False, reason) reason = response["result"].get("reason", "Ok") return AuthorizationOutcome(True, reason) @staticmethod def _extract_service_from_path(path: str) -> str | None: """If a request path contains a service name, extract it. Example: /api/jobs/someservice.instance/110/run -> someservice :param str path: request path :return: service name, or None if not found """ match = SERVICE_NAME_PATH_PATTERN.search(path) return match.group(1) if match else None ================================================ FILE: tron/api/controller.py ================================================ """ Web Controllers for the API. """ import logging from typing import TYPE_CHECKING from typing import TypedDict from tron import yaml from tron.config.manager import ConfigManager from tron.core.actionrun import ActionRun from tron.core.jobrun import JobRun from tron.eventbus import EventBus if TYPE_CHECKING: from tron.mcp import MasterControlProgram log = logging.getLogger(__name__) class UnknownCommandError(Exception): """Exception raised when a controller received an unknown command.""" class InvalidCommandForActionState(Exception): """ Exception raised when a controller attempts a command on an action in a state that does not support that command (e.g., skipping a successful run). """ def __init__(self, command: str, action_name: str, action_state: str) -> None: self.command = command self.action_name = action_name self.action_state = action_state self.message = f"Failed to {command} on {action_name}. State is {action_state}." super().__init__() class JobCollectionController: def __init__(self, job_collection): self.job_collection = job_collection def handle_command(self, command, old_name=None, new_name=None): if command == "move": if old_name not in self.job_collection.get_names(): return f"Error: {old_name} doesn't exist" if new_name in self.job_collection.get_names(): return f"Error: {new_name} exists already" return self.job_collection.move(old_name, new_name) raise UnknownCommandError(f"Unknown command {command}. Try running this on an individual job or action run id") class ActionRunController: mapped_commands = { "start", "success", "cancel", "fail", "skip", "stop", "kill", "retry", "recover", } def __init__(self, action_run: ActionRun, job_run: JobRun) -> None: self.action_run = action_run self.job_run = job_run def handle_command(self, command, **kwargs): if command not in self.mapped_commands: raise UnknownCommandError( f"Unknown command {command}. You can only do one of the following to Action runs: {self.mapped_commands}" ) if command == "start" and self.job_run.is_scheduled: return "Action run cannot be started if its job run is still " "scheduled." if command == "recover" and not self.action_run.is_unknown: return "Action run cannot be recovered if its state is not unknown." if command in ("stop", "kill"): return self.handle_termination(command) if command == "retry": original_command = not kwargs.get("use_latest_command", False) return self.handle_retry(original_command) if getattr(self.action_run, command)(): msg = "%s now in state %s" return msg % (self.action_run, self.action_run.state) raise InvalidCommandForActionState( command=command, action_name=self.action_run.name, action_state=self.action_run.state, ) def handle_termination(self, command): try: # Extra message is only used for killing mesos action as warning so far. extra_msg = getattr(self.action_run, command)() msg = "Attempting to %s %s" if extra_msg is not None: msg = msg + "\n" + extra_msg return msg % (command, self.action_run) except NotImplementedError as e: msg = "Failed to %s: %s" return msg % (command, e) def handle_retry(self, original_command): cleanup_run = self.job_run.action_runs.cleanup_action_run if cleanup_run and cleanup_run.is_done: return "JobRun has run a cleanup action, use rerun instead" if self.action_run.retry(original_command=original_command): return "Retrying %s" % self.action_run else: return "Failed to schedule retry for %s" % self.action_run class JobRunController: mapped_commands = {"start", "success", "cancel", "fail", "stop"} def __init__(self, job_run, job_scheduler): self.job_run = job_run self.job_scheduler = job_scheduler def handle_command(self, command): # as of TRON-1340, `tronctl backfill` depends on this response value # (i.e. "Created JobRun:"). be careful when changing this! if command == "restart" or command == "rerun": runs = self.job_scheduler.manual_start(self.job_run.run_time) return "Created %s" % ",".join(str(run) for run in runs) if command in self.mapped_commands: if getattr(self.job_run, command)(): return f"{self.job_run} now in state {self.job_run.state}" msg = "Failed to %s, %s in state %s" return msg % (command, self.job_run, self.job_run.state) if command == "retry": raise UnknownCommandError( "Error: Job runs cannot be retried, only individual actions can. Did you mean 'rerun'?" ) else: raise UnknownCommandError( f"Unknown command {command}. Only one of the following applies to a Job run: {self.mapped_commands}" ) class JobController: def __init__(self, job_scheduler): self.job_scheduler = job_scheduler def handle_command(self, command, run_time=None): if command == "enable": self.job_scheduler.enable() return "%s is enabled" % self.job_scheduler.get_job() elif command == "disable": self.job_scheduler.disable() return "%s is disabled" % self.job_scheduler.get_job() elif command == "start": runs = self.job_scheduler.manual_start(run_time=run_time) return "Created %s" % ",".join(str(run) for run in runs) if command == "retry": raise UnknownCommandError( "Error: A whole Job cannot be retried, only individual actions for a specific job run id can." ) elif command in ["stop", "success", "cancel", "fail", "stop"]: raise UnknownCommandError( f"Error: {command} doesn't apply to a whole Job. Please run this on an individual job run id. Hint: try '{self.job_scheduler.get_job()}.-1' for the latest job id" ) else: raise UnknownCommandError( f"Unknown command {command}. Does it apply to a whole job? Try a specific Job id or individual action" ) class ConfigResponse(TypedDict): config: str hash: str class ConfigController: """Control config. Return config contents and accept updated configuration from the API. """ DEFAULT_NAMED_CONFIG = "\njobs:\n" def __init__(self, mcp: "MasterControlProgram") -> None: self.mcp = mcp self.config_manager: ConfigManager = mcp.get_config_manager() def _get_config_content(self, name: str) -> str: if name not in self.config_manager: return self.DEFAULT_NAMED_CONFIG return self.config_manager.read_raw_config(name) def read_config(self, name: str) -> ConfigResponse: config_content = self._get_config_content(name) config_hash = self.config_manager.get_hash(name) return {"config": config_content, "hash": config_hash} def read_all_configs(self) -> dict[str, ConfigResponse]: configs = {} for service in self.config_manager.get_namespaces(): config: ConfigResponse = { "config": self._get_config_content(service), "hash": self.config_manager.get_hash(service), } configs[service] = config return configs def check_config(self, name, content, config_hash): """Update a configuration fragment and reload the MCP.""" if self.config_manager.get_hash(name) != config_hash: return "Configuration update will fail: config is stale, try again" try: content = yaml.load(content) self.config_manager.validate_with_fragment(name, content) except Exception as e: return "Configuration update will fail: %s" % str(e) def update_config(self, name, content, config_hash): """Update a configuration fragment and reload the MCP.""" if self.config_manager.get_hash(name) != config_hash: return "Configuration has changed. Please try again." old_config = self.read_config(name)["config"] try: log.info(f"Reconfiguring namespace {name}") self.config_manager.write_config(name, content) self.mcp.reconfigure(namespace=name) except Exception as e: log.error(f"Configuration for {name} update failed: {e}") log.error("Reconfiguring with the previous good configuration") try: self.config_manager.write_config(name, old_config) self.mcp.reconfigure(namespace=name) except Exception as e: log.error("Could not restore old config: %s" % e) return str(e) return str(e) def delete_config(self, name, content, config_hash): """Delete a configuration fragment and reload the MCP.""" if self.config_manager.get_hash(name) != config_hash: return "Configuration has changed. Please try again." if content != "": return "Configuration content is not empty, will not delete." try: log.info(f"Deleting namespace {name}") self.config_manager.delete_config(name) self.mcp.reconfigure(namespace=name) except Exception as e: log.error(f"Deleting configuration for {name} failed: {e}") return str(e) def get_namespaces(self): return self.config_manager.get_namespaces() class EventsController: COMMANDS = {"publish", "discard"} def publish(self, event): if not EventBus.instance: return dict(error="EventBus disabled") if EventBus.has_event(event): msg = f"event {event} already published" log.warning(msg) return dict(response=msg) if not EventBus.publish(event): msg = f"could not publish {event}" log.error(msg) return dict(error=msg) return dict(response="OK") def discard(self, event): if not EventBus.instance: return dict(error="EventBus disabled") if not EventBus.discard(event): msg = f"could not discard {event}" log.error(msg) return dict(error=msg) return dict(response="OK") def info(self): if not EventBus.instance: return dict(error="EventBus disabled") return dict(response=EventBus.instance.event_log) ================================================ FILE: tron/api/requestargs.py ================================================ """Functions for returning validated values from a twisted.web.Request object. """ import datetime DATE_FORMAT = "%Y-%m-%d %H:%M:%S" def get_integer(request, key): """Returns the first value in the request args for the given key, if that value is an integer. Otherwise returns None. """ value = get_string(request, key) if value is None or not value.isdigit(): return None return int(value) def get_string(request, key): """Returns the first value in the request args for a given key.""" if not request.args: return None if type(key) is not bytes: key = key.encode() if key not in request.args: return None val = request.args[key][0] if val is not None and type(val) is bytes: val = val.decode() return val def get_bool(request, key, default=None): """Returns True if the key exists and is truthy in the request args.""" int_value = get_integer(request, key) if int_value is None: return default return bool(int_value) def get_datetime(request, key): """Returns the first value in the request args for a given key. Casts to a datetime. Returns None if the value cannot be converted to datetime. """ val = get_string(request, key) if not val: return None try: return datetime.datetime.strptime(val, DATE_FORMAT) except ValueError: return None ================================================ FILE: tron/api/resource.py ================================================ """ Web Services Interface used by command-line clients and web frontend to view current state, event history and send commands to trond. """ import collections import datetime import json import logging import traceback import staticconf from prometheus_client.twisted import MetricsResource as MetricsResourceProm from twisted.web import http from twisted.web import resource from twisted.web import server from twisted.web import static from tron import __version__ from tron.api import adapter from tron.api import controller from tron.api import requestargs from tron.api.async_resource import AsyncResource from tron.api.auth import AuthorizationFilter from tron.config.static_config import get_config_watcher from tron.config.static_config import NAMESPACE from tron.metrics import meter from tron.metrics import view_all_metrics from tron.utils import maybe_decode log = logging.getLogger(__name__) class JSONEncoder(json.JSONEncoder): """Custom JSON for certain objects""" def default(self, o): if isinstance(o, datetime.datetime): return o.strftime("%Y-%m-%d %H:%M:%S") if isinstance(o, datetime.date): return o.isoformat() if isinstance(o, collections.abc.KeysView): return list(o) return super().default(o) def respond(request, response, code=None, headers=None): """Helper to generate a json response""" if code is None: if type(response) is dict and response.get("error"): code = http.INTERNAL_SERVER_ERROR else: code = http.OK request.setResponseCode(code) request.setHeader(b"content-type", b"application/json; charset=utf-8") request.setHeader(b"Access-Control-Allow-Origin", b"*") for key, val in (headers or {}).items(): request.setHeader(str(key), str(val)) result = ( json.dumps( response, cls=JSONEncoder, ) if response else "" ) if type(result) is not bytes: result = result.encode("utf8") return result def handle_command(request, api_controller, obj, **kwargs): """Handle a request to perform a command.""" command = requestargs.get_string(request, "command") log.info("Handling '%s' request on %s", command, obj) try: response = api_controller.handle_command(command, **kwargs) return respond(request=request, response={"result": response}) except controller.UnknownCommandError: error_msg = f"Unknown command '{command}' for '{obj}'" log.warning(error_msg) return respond( request=request, response={"error": error_msg}, code=http.NOT_IMPLEMENTED, ) except controller.InvalidCommandForActionState as e: log.warning(e.message) return respond( request=request, response={"error": e.message}, code=http.CONFLICT, ) except Exception as e: log.exception("%r while executing command %s for %s", e, command, obj) trace = traceback.format_exc() return respond(request=request, response={"error": trace}) class AuthenticatedResource(resource.Resource): def render(self, request): """Overriding base `render` method to support authentication""" auth_outcome = AuthorizationFilter.get_from_env().is_request_authorized(request) if not auth_outcome.authorized: return respond( request=request, response={"error": f"Auth failed (reason: {auth_outcome.reason})"}, code=http.FORBIDDEN, headers={"X-Auth-Failure-Reason": auth_outcome.reason}, ) return super().render(request) class ErrorResource(resource.Resource): """Equivalent to resource.NoResource, except error message is returned as JSON, not HTML""" def __init__(self, error="No Such Resource", code=http.NOT_FOUND): resource.Resource.__init__(self) self.code = code self.error = error @AsyncResource.bounded def render_GET(self, request): return respond(request=request, response={"error": self.error}, code=self.code) @AsyncResource.exclusive def render_POST(self, request): return respond(request=request, response={"error": self.error}, code=self.code) def getChild(self, chnam, request): """Overrided getChild to ensure a NoResource is not returned""" return self def resource_from_collection(collection, name, child_resource): """Return a child resource from a collection by name. If no item is found, return ErrorResource. """ item = collection.get_by_name(name) if item is None: return ErrorResource("Cannot find child '%s'" % name) return child_resource(item) class ActionRunResource(AuthenticatedResource): isLeaf = True def __init__(self, action_run, job_run): resource.Resource.__init__(self) self.action_run = action_run self.job_run = job_run self.controller = controller.ActionRunController(action_run, job_run) self.config_watcher = get_config_watcher() @AsyncResource.bounded def render_GET(self, request): run_adapter = adapter.ActionRunAdapter( self.action_run, self.job_run, requestargs.get_integer(request, "num_lines") or staticconf.read("logging.max_lines_to_display", namespace=NAMESPACE), include_stdout=requestargs.get_bool(request, "include_stdout"), include_stderr=requestargs.get_bool(request, "include_stderr"), include_meta=requestargs.get_bool(request, "include_meta"), ) return respond(request=request, response=run_adapter.get_repr()) @AsyncResource.exclusive def render_POST(self, request): use_latest_command = requestargs.get_bool(request, "use_latest_command", False) return handle_command( request, self.controller, self.action_run, use_latest_command=use_latest_command, ) class JobRunResource(AuthenticatedResource): def __init__(self, job_run, job_scheduler): resource.Resource.__init__(self) self.job_run = job_run self.job_scheduler = job_scheduler self.controller = controller.JobRunController(job_run, job_scheduler) def getChild(self, action_name, _): if not action_name: return self action_name = maybe_decode( action_name ) # TODO: TRON-2293 maybe_decode is a relic of Python2->Python3 migration. Remove it. if action_name in self.job_run.action_runs: action_run = self.job_run.action_runs[action_name] return ActionRunResource(action_run, self.job_run) return ErrorResource( f"Cannot find action '{action_name}' for " f"'{self.job_run}'", ) @AsyncResource.bounded def render_GET(self, request): include_runs = requestargs.get_bool(request, "include_action_runs") include_graph = requestargs.get_bool(request, "include_action_graph") run_adapter = adapter.JobRunAdapter( self.job_run, include_action_runs=include_runs, include_action_graph=include_graph, ) return respond(request=request, response=run_adapter.get_repr()) @AsyncResource.exclusive def render_POST(self, request): return handle_command(request, self.controller, self.job_run) def is_negative_int(string): return string.startswith("-") and string[1:].isdigit() class JobResource(AuthenticatedResource): def __init__(self, job_scheduler): resource.Resource.__init__(self) self.job_scheduler = job_scheduler self.controller = controller.JobController(job_scheduler) def get_run_from_identifier(self, run_id): job_runs = self.job_scheduler.get_job_runs() if run_id.upper() == "HEAD": return job_runs.get_newest() if run_id.isdigit(): return job_runs.get_run_by_num(int(run_id)) if is_negative_int(run_id): return job_runs.get_run_by_index(int(run_id)) def getChild(self, run_id, _): if not run_id: return self run_id = maybe_decode( run_id ) # TODO: TRON-2293 maybe_decode is a relic of Python2->Python3 migration. Remove it. run = self.get_run_from_identifier(run_id) if run: return JobRunResource(run, self.job_scheduler) job = self.job_scheduler.get_job() if run_id in job.action_graph.names(): action_runs = job.runs.get_action_runs(run_id) return ActionRunHistoryResource(action_runs) return ErrorResource(f"Cannot find job run '{run_id}' for '{job}'") @AsyncResource.bounded def render_GET(self, request): include_action_runs = requestargs.get_bool( request, "include_action_runs", ) include_graph = requestargs.get_bool(request, "include_action_graph") num_runs = requestargs.get_integer(request, "num_runs") job_adapter = adapter.JobAdapter( self.job_scheduler.get_job(), include_job_runs=True, include_action_runs=include_action_runs, include_action_graph=include_graph, num_runs=num_runs, ) return respond(request=request, response=job_adapter.get_repr()) @AsyncResource.exclusive def render_POST(self, request): run_time = requestargs.get_datetime(request, "run_time") return handle_command( request, self.controller, self.job_scheduler, run_time=run_time, ) class ActionRunHistoryResource(AuthenticatedResource): isLeaf = True def __init__(self, action_runs): resource.Resource.__init__(self) self.action_runs = action_runs @AsyncResource.bounded def render_GET(self, request): return respond( request=request, response=adapter.adapt_many(adapter.ActionRunAdapter, self.action_runs), ) class JobCollectionResource(AuthenticatedResource): def __init__(self, job_collection): self.job_collection = job_collection self.controller = controller.JobCollectionController(job_collection) resource.Resource.__init__(self) def getChild(self, name, request): if not name: return self name = maybe_decode(name) # TODO: TRON-2293 maybe_decode is a relic of Python2->Python3 migration. Remove it. return resource_from_collection(self.job_collection, name, JobResource) def get_data( self, include_job_run=False, include_action_runs=False, include_action_graph=True, include_node_pool=True, ): return adapter.adapt_many( adapter.JobAdapter, self.job_collection.get_jobs(), include_job_run, include_action_runs, include_action_graph, include_node_pool, num_runs=5, ) def get_job_index(self): jobs = adapter.adapt_many( adapter.JobIndexAdapter, self.job_collection.get_jobs(), ) return {job["name"]: job["actions"] for job in jobs} @AsyncResource.bounded def render_GET(self, request): include_job_runs = requestargs.get_bool( request, "include_job_runs", default=False, ) include_action_runs = requestargs.get_bool( request, "include_action_runs", default=False, ) include_action_graph = requestargs.get_bool( request, "include_action_graph", default=True, ) include_node_pool = requestargs.get_bool( request, "include_node_pool", default=True, ) response = dict( jobs=self.get_data( include_job_runs, include_action_runs, include_action_graph, include_node_pool, ), ) return respond(request=request, response=response) @AsyncResource.exclusive def render_POST(self, request): old_name = requestargs.get_string(request, "old_name") new_name = requestargs.get_string(request, "new_name") return handle_command( request=request, api_controller=self.controller, obj=self.job_collection, old_name=old_name, new_name=new_name, ) class ConfigResource(AuthenticatedResource): """Resource for configuration changes""" isLeaf = True def __init__(self, master_control): self.controller = controller.ConfigController(master_control) resource.Resource.__init__(self) def get_config_index(self): return self.controller.get_namespaces() @AsyncResource.bounded def render_GET(self, request): config_name = requestargs.get_string(request, "name") if not config_name: response = self.controller.read_all_configs() else: response = self.controller.read_config(config_name) return respond(request=request, response=response) @AsyncResource.exclusive def render_POST(self, request): config_content = requestargs.get_string(request, "config") name = requestargs.get_string(request, "name") config_hash = requestargs.get_string(request, "hash") check = requestargs.get_bool(request, "check") if not name: return respond( request=request, response={"error": "'name' for config is required."}, code=http.BAD_REQUEST, ) response = {"status": "Active"} if check: fn = self.controller.check_config req = "configure check" elif config_content == "": fn = self.controller.delete_config req = "configuration delete" else: fn = self.controller.update_config req = "reconfigure" log.info(f"Handling {req} request: {name}, {config_hash}") error = fn(name, config_content, config_hash) if error: response["error"] = error return respond(request=request, response=response) class StatusResource(resource.Resource): isLeaf = True def __init__(self, master_control): self._master_control = master_control resource.Resource.__init__(self) @AsyncResource.bounded def render_GET(self, request): return respond( request=request, response={ "status": "I'm alive.", "version": __version__, "boot_time": int(self._master_control.boot_time), }, ) class MetricsResource(resource.Resource): isLeaf = True def __init__(self): resource.Resource.__init__(self) @AsyncResource.exclusive def render_GET(self, request): return respond(request=request, response=view_all_metrics()) class EventsResource(AuthenticatedResource): isLeaf = True def __init__(self): super().__init__() self.controller = controller.EventsController() @AsyncResource.exclusive def render_GET(self, request): response = self.controller.info() return respond(request=request, response=response) @AsyncResource.bounded def render_POST(self, request): command = requestargs.get_string(request, "command") if command not in self.controller.COMMANDS: return respond( request=request, response=dict(error=f"Unknown command: {command}"), code=http.BAD_REQUEST, ) event = requestargs.get_string(request, "event") fn = getattr(self.controller, command) response = fn(event) return respond(request=request, response=response) class ApiRootResource(AuthenticatedResource): def __init__(self, mcp): self._master_control = mcp resource.Resource.__init__(self) # Setup children self.putChild( b"jobs", JobCollectionResource(mcp.get_job_collection()), ) self.putChild(b"config", ConfigResource(mcp)) self.putChild(b"status", StatusResource(mcp)) self.putChild(b"events", EventsResource()) self.putChild(b"metrics", MetricsResource()) self.putChild(b"prom-metrics", MetricsResourceProm()) self.putChild(b"", self) @AsyncResource.bounded def render_GET(self, request): """Return an index of urls for resources.""" response = { "jobs": self.children[b"jobs"].get_job_index(), "namespaces": self.children[b"config"].get_config_index(), } return respond(request=request, response=response) class RootResource(resource.Resource): def __init__(self, mcp, web_path): resource.Resource.__init__(self) self.web_path = web_path self.mcp = mcp self.putChild(b"api", ApiRootResource(self.mcp)) self.putChild(b"web", static.File(web_path)) self.putChild(b"", self) def render_GET(self, request): request.redirect(b"/web") request.finish() return server.NOT_DONE_YET def __str__(self): return f"{type(self).__name__}({self.mcp}, {self.web_path})" class LogAdapter: def __init__(self, logger): self.logger = logger def write(self, line): self.logger.info(line.rstrip(b"\n")) def close(self): pass class TronSite(server.Site): """Subclass of a twisted Site to customize logging.""" access_log = logging.getLogger("tron.api.www.access") @classmethod def create(cls, mcp, web_path): return cls(RootResource(mcp, web_path)) def startFactory(self): server.Site.startFactory(self) self.logFile = LogAdapter(self.access_log) def log(self, request): super().log(request) if 200 <= request.code < 300: meter("tron.site.2xx") if 300 <= request.code < 400: meter("tron.site.3xx") if 400 <= request.code < 500: meter("tron.site.4xx") if 500 <= request.code < 600: meter("tron.site.5xx") def __repr__(self): return f"{self.__class__.__name__}({self.resource})" ================================================ FILE: tron/bin/action_runner.py ================================================ #!/usr/bin/env python3.10 """ Write pid and stdout/stderr to a standard location before execing a command. """ import argparse import contextlib import logging import os import subprocess import sys import threading import time from tron import yaml STATUS_FILE = "status" class StatusFile: """Manage a status file.""" def __init__(self, filename): self.filename = filename def get_content(self, run_id, command, proc): return { "run_id": run_id, "command": command, "pid": proc.pid, "return_code": proc.returncode, "runner_pid": os.getpid(), "timestamp": time.time(), } @contextlib.contextmanager def wrap(self, command, run_id, proc): with open(self.filename, "w") as fh: yaml.safe_dump( self.get_content( run_id=run_id, command=command, proc=proc, ), fh, explicit_start=True, width=1000000, ) try: yield finally: with open(self.filename, "a") as fh: yaml.safe_dump( self.get_content( run_id=run_id, command=command, proc=proc, ), fh, explicit_start=True, width=1000000, ) def validate_output_dir(path): if os.path.isdir(path): if not os.access(path, os.W_OK): raise OSError("Output dir %s not writable" % path) return else: try: os.makedirs(path) except OSError: raise OSError("Could not create output dir %s" % path) def build_environment(run_id, original_env=None): if original_env is None: original_env = dict(os.environ) try: namespace, job, run_num, action = run_id.split(".", maxsplit=3) except ValueError: # if we can't parse the run_id, we don't want to abort, so just # set these semi-arbitrarily namespace, job, run_num, action = ["UNKNOWN"] * 4 new_env = dict(original_env) new_env["TRON_JOB_NAMESPACE"] = namespace new_env["TRON_JOB_NAME"] = job new_env["TRON_RUN_NUM"] = run_num new_env["TRON_ACTION"] = action logging.debug(new_env) return new_env def build_labels( run_id: str, original_labels: dict[str, str] | None = None, attempt_number: int | None = None, ) -> dict[str, str]: if original_labels is None: original_labels = {} try: # reminder: the format here is "namespace.job.run_num.action" _, _, run_num, _ = run_id.split(".", maxsplit=3) except ValueError: # if we can't parse the run_id, we don't want to abort, so just # set these semi-arbitrarily run_num = "UNKNOWN" new_labels = dict(original_labels) new_labels["tron.yelp.com/run_num"] = run_num if attempt_number is not None: new_labels["tron.yelp.com/attempt_number"] = str(attempt_number) return new_labels def run_proc(output_path, command, run_id, proc): logging.warning(f"{run_id} running as pid {proc.pid}") status_file = StatusFile(os.path.join(output_path, STATUS_FILE)) with status_file.wrap( command=command, run_id=run_id, proc=proc, ): returncode = proc.wait() logging.warning(f"pid {proc.pid} exited with returncode {returncode}") return returncode def parse_args(): parser = argparse.ArgumentParser(description="Action Runner for Tron") parser.add_argument( "output_dir", help="The directory to store the state of the action run", ) parser.add_argument( "command", help="the command to run", ) parser.add_argument( "run_id", help="run_id of the action", ) return parser.parse_args() def run_command(command, run_id): return subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=build_environment(run_id=run_id), ) def stream(source, dst): is_connected = True logging.warning(f"streaming {source.name} to {dst.name}") for line in iter(source.readline, b""): if is_connected: try: dst.write(line.decode("utf-8")) dst.flush() logging.warning(f"{dst.name}: {line}") except Exception as e: logging.warning(f"failed writing to {dst}: {e}") logging.warning(f"{dst.name}: {line}") is_connected = False else: logging.warning(f"{dst.name}: {line}") is_connected = False def configure_logging(run_id, output_dir): output_file = os.path.join(output_dir, f"{run_id}-{os.getpid()}.log") logging.basicConfig( filename=output_file, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%S%z", ) def main(): args = parse_args() validate_output_dir(args.output_dir) configure_logging(run_id=args.run_id, output_dir=args.output_dir) proc = run_command(command=args.command, run_id=args.run_id) threads = [ threading.Thread(target=stream, args=p, daemon=True) for p in [(proc.stdout, sys.stdout), (proc.stderr, sys.stderr)] ] for t in threads: t.start() returncode = run_proc( output_path=args.output_dir, run_id=args.run_id, command=args.command, proc=proc, ) for t in threads: t.join() return returncode if __name__ == "__main__": sys.exit(main()) ================================================ FILE: tron/bin/action_status.py ================================================ #!/usr/bin/env python3.10 import argparse import logging import os import signal from tron import yaml log = logging.getLogger("tron.action_status") STATUS_FILE = "status" def get_field(field, status_file): docs = yaml.load_all(status_file.read()) content = list(docs)[-1] return content.get(field) def print_status_file(status_file): for line in status_file.readlines(): print(yaml.load(line)) def send_signal(signal_num, status_file): pid = get_field("pid", status_file) if pid: try: os.killpg(os.getpgid(pid), signal_num) except OSError as e: msg = "Failed to signal %s with %s: %s" raise SystemExit(msg % (pid, signal_num, e)) commands = { "print": print_status_file, "pid": lambda statusfile: print(get_field("pid", statusfile)), "return_code": lambda statusfile: print(get_field("return_code", statusfile)), "terminate": lambda statusfile: send_signal(signal.SIGTERM, statusfile), "kill": lambda statusfile: send_signal(signal.SIGKILL, statusfile), } def parse_args(): parser = argparse.ArgumentParser(description="Action Status for Tron") parser.add_argument( "output_dir", help="The directory where the state of the action run is", ) parser.add_argument( "command", help="the command to run", ) parser.add_argument( "run_id", help="run_id of the action", ) return parser.parse_args() def run_command(command, status_file): commands[command](status_file) def main(): logging.basicConfig() args = parse_args() with open(os.path.join(args.output_dir, STATUS_FILE)) as f: run_command(args.command, f) if __name__ == "__main__": main() ================================================ FILE: tron/bin/check_tron_datastore_staleness.py ================================================ #!/usr/bin/env python3.10 import argparse import logging import os import sys import time import pytz from tron.config import manager from tron.config import schema from tron.serialize.runstate.statemanager import PersistenceManagerFactory # Default values for arguments DEFAULT_WORKING_DIR = "/var/lib/tron/" DEFAULT_CONF_PATH = "config/" DEFAULT_STALENESS_THRESHOLD = 1800 log = logging.getLogger("check_tron_datastore_staleness") def get_last_run_time(job): """ Get all sorted timestamps, and only count the actions that actually ran """ timestamps = [] job_runs = job["runs"] for run in job_runs: for action in run["runs"]: if action.get("start_time") and action.get("state") != "scheduled": timestamps.append(action.get("start_time")) return max(timestamps) if timestamps else None def parse_cli(): parser = argparse.ArgumentParser() parser.add_argument( "-w", "--working-dir", default=DEFAULT_WORKING_DIR, help="Working directory for the Tron daemon, default %(default)s", ) parser.add_argument( "-c", "--config-path", default=DEFAULT_CONF_PATH, help="File path to the Tron configuration file", ) parser.add_argument( "--job-name", required=True, help="The job name to read timestamp from", ) parser.add_argument( "--staleness-threshold", default=DEFAULT_STALENESS_THRESHOLD, help="how long (in seconds) to wait to alert after the last timestamp of the job", ) args = parser.parse_args() args.working_dir = os.path.abspath(args.working_dir) args.config_path = os.path.join( args.working_dir, args.config_path, ) return args def read_config(args): return manager.ConfigManager(args.config_path).load().get_master().state_persistence def main(): # Fetch configs. You can find the arguments in puppet. args = parse_cli() persistence_config = read_config(args) store_type = schema.StatePersistenceTypes(persistence_config.store_type) job_name = args.job_name # Alert for DynamoDB if store_type == schema.StatePersistenceTypes.dynamodb: # Fetch job state from dynamodb state_manager = PersistenceManagerFactory.from_config(persistence_config) try: job = state_manager.restore(job_names=[job_name])["job_state"][job_name] except Exception as e: logging.exception(f"UNKN: Failed to retreive status for job {job_name} due to {e}") sys.exit(3) # Exit if the job never runs. last_run_time = get_last_run_time(job) if not last_run_time: logging.error( f"WARN: No last run for {job_name} found. If the job was just added, it might take some time for it to run" ) sys.exit(1) # Alert if timestamp is not updated after staleness_threshold stateless_for_secs = time.time() - last_run_time.astimezone(pytz.utc).timestamp() if stateless_for_secs > args.staleness_threshold: logging.error(f"CRIT: {job_name} has not been updated in DynamoDB for {stateless_for_secs} seconds") sys.exit(2) else: logging.info(f"OK: DynamoDB is up to date. It's last updated at {last_run_time}") sys.exit(0) # Alert for BerkeleyDB elif store_type == schema.StatePersistenceTypes.shelve: os.execl( "/usr/lib/nagios/plugins/check_file_age", "/nail/tron/tron_state", "-w", str(args.staleness_threshold), "-c", str(args.staleness_threshold), ) else: logging.exception(f"UNKN: Not designed to check this type of datastore: {store_type}") sys.exit(3) if __name__ == "__main__": main() ================================================ FILE: tron/bin/check_tron_jobs.py ================================================ #!/usr/bin/env python3.10 import datetime import logging import pprint import sys import time from collections import defaultdict from enum import Enum import pytimeparse # type: ignore[import-untyped] # no stubs or py.typed marker; likely want to move off of this from pyrsistent import m from pyrsistent import pmap from pysensu_yelp import send_event from tron.commands import cmd_utils from tron.commands import display from tron.commands.client import Client from tron.commands.client import get_object_type_from_identifier from tron.utils.logreader import get_superregion PRECIOUS_JOB_ATTR = "check_that_every_day_has_a_successful_run" NUM_PRECIOUS = 7 log = logging.getLogger("check_tron_jobs") _run_interval = None class State(Enum): SUCCEEDED = "succeeded" FAILED = "failed" STUCK = "stuck" NO_RUN_YET = "no_run_yet" NO_RUNS_TO_CHECK = "no_runs_to_check" UNKNOWN = "unknown" SKIPPED = "skipped" def parse_cli(): parser = cmd_utils.build_option_parser() parser.add_argument( "--dry-run", action="store_true", default=False, help="Don't actually send alerts out. Defaults to %(default)s", ) parser.add_argument( "--job", default=None, help="Check a particular job. If unset checks all jobs", ) parser.add_argument( "--run-interval", help="Run interval for this monitoring script. This is used to " "calculate realert and alert_after setting. " "Default to %(default)s (seconds)", type=int, dest="run_interval", default=300, ) parser.add_argument( "--skip-sensu-failure-logging", help="Skip including stdout/stderr logs in alerts for failed jobs", action="store_true", dest="skip_sensu_failure_logging", default=False, ) args = parser.parse_args() return args def _timestamp_to_timeobj(timestamp): return time.strptime(timestamp, "%Y-%m-%d %H:%M:%S") def _timestamp_to_shortdate(timestamp, separator="."): return time.strftime( "%Y{0}%m{0}%d".format(separator), _timestamp_to_timeobj(timestamp), ) def compute_check_result_for_job_runs(client, job, job_content, url_index, hide_stderr=False): cluster = client.cluster_name kwargs = {} if job_content is None: kwargs["output"] = f"OK: {job['name']} was just added and hasn't run yet on {cluster}." kwargs["status"] = 0 return kwargs relevant_job_run, last_state = get_relevant_run_and_state(job_content) if relevant_job_run is None: kwargs["output"] = ( f"CRIT: {job['name']} hasn't had a successful " f"run yet on {cluster}.\n{pretty_print_job(job_content)}" ) kwargs["status"] = 2 return kwargs else: # if no run scheduled, no run_time available relevant_job_run_date = _timestamp_to_shortdate( relevant_job_run["run_time"], ) # A job_run is like MASTER.foo.1 job_run_id = relevant_job_run["id"] # A job action is like MASTER.foo.1.step1 actions_expected_runtime = job_content.get("actions_expected_runtime", {}) relevant_action = get_relevant_action( action_runs=relevant_job_run["runs"], last_state=last_state, actions_expected_runtime=actions_expected_runtime, ) action_run_id = get_object_type_from_identifier( url_index, relevant_action["id"], ) if last_state in (State.STUCK, State.FAILED, State.UNKNOWN): if _skip_sensu_failure_logging: job_run_url = "/".join(job_run_id.rsplit(".", 1)) tronweb_url = f"http://y/tron-{get_superregion()}/#job/{job_run_url}" stderr_default = f"Please visit {tronweb_url} for stderr details." action_run_details = {} else: stderr_default = "(No stderr available)" action_run_details = client.action_runs(action_run_id.url, num_lines=10) else: action_run_details = {} if last_state == State.SUCCEEDED: prefix = f"OK: The last job ({job_run_id}) run succeeded on {cluster}. Will watch future or in progress runs for the next failure" status = 0 stderr = "" elif last_state == State.NO_RUNS_TO_CHECK: prefix = f"OK: The job {job['name']} is new and/or has no runs to check on {cluster}" status = 0 stderr = "" elif last_state == State.SKIPPED: prefix = f"OK: The last job ({job_run_id}) run was skipped on {cluster}. Will watch future or in progress runs for the next failure" status = 0 stderr = "" elif last_state == State.STUCK: if job["monitoring"].get("page_for_expected_runtime", False): level = "CRIT" status = 2 else: level = "WARN" status = 1 prefix = f"{level}: Job {job_run_id} exceeded expected runtime or still running when next job is scheduled on {cluster}" stderr = "\n".join(action_run_details.get("stderr", [stderr_default])) elif last_state == State.FAILED: prefix = f"CRIT: The last job run ({job_run_id}) failed on {cluster}!" status = 2 stderr = "\n".join(action_run_details.get("stderr", [stderr_default])) elif last_state == State.UNKNOWN: prefix = f"CRIT: Job {job_run_id} has gone 'unknown' and might need manual intervention on {cluster}" status = 2 stderr = "" else: prefix = f"UNKNOWN: Job {job_run_id} is in a state that check_tron_jobs doesn't understand" status = 3 stderr = "" if hide_stderr: stderr = "" precious_runs_note = "" if job["monitoring"].get(PRECIOUS_JOB_ATTR, False) and status != 0: precious_runs_note = f"Note: This alert is the run for {relevant_job_run_date}. A resolve event will not occur until a job run for this date succeeds.\n" kwargs["output"] = ( f"{prefix}\n" f"{stderr}\n" f"The latest run, {relevant_job_run['id']} {relevant_job_run['state']}\n" f"{precious_runs_note}" ) if action_run_details: kwargs["output"] += "\nHere is the last action:\n" f"{pretty_print_actions(action_run_details)}\n\n" kwargs["output"] += ( "And the job run view:\n" f"{pretty_print_job_run(relevant_job_run)}\n\n" "Here is the whole job view for context:\n" f"{pretty_print_job(job_content)}" ) kwargs["status"] = status return kwargs def pretty_print_job(job_content): return display.format_job_details(job_content) def pretty_print_job_run(job_run): display_action = display.DisplayActionRuns() return display_action.format(job_run) def pretty_print_actions(action_run): return display.format_action_run_details(action_run) def get_relevant_run_and_state(job_content): # The order of job run to check is as follows: # 1. The scheduled but hasn't run one checked first # 2. Then currently running ones are always checked (in case an action is failed/unknown) # 3. If there are multiple running ones, then most recent run_time wins # 4. If nothing is currently running, then most recent end_time wins job_runs = sorted( job_content.get("runs", []), key=lambda k: (k["end_time"] is None, k["end_time"], k["run_time"]), reverse=True, ) if len(job_runs) == 0: return None, State.NO_RUN_YET job_expected_runtime = job_content.get("expected_runtime", None) actions_expected_runtime = job_content.get("actions_expected_runtime", {}) stuck_run = is_job_stuck( job_runs=job_runs, job_expected_runtime=job_expected_runtime, actions_expected_runtime=actions_expected_runtime, allow_overlap=job_content.get("allow_overlap", False), queueing=job_content.get("queueing", True), ) for run in job_runs: state = run.get("state", "unknown") if state in ["failed", "succeeded", "unknown", "skipped"]: return run, State(state) elif state in ["running", "waiting", "starting"]: action_state = is_action_failed_or_unknown(run) if action_state != State.SUCCEEDED: return run, action_state elif stuck_run is not None: return stuck_run, State.STUCK return job_runs[0], State.NO_RUNS_TO_CHECK def is_action_failed_or_unknown(job_run): for run in job_run.get("runs", []): if run.get("state", None) in ["failed", "unknown"]: return State(run.get("state")) return State.SUCCEEDED def is_job_stuck( job_runs, job_expected_runtime, actions_expected_runtime, allow_overlap, queueing, ): next_run_time = None for job_run in job_runs: states_to_check = {"running", "waiting", "starting"} if job_run.get("state", "unknown") in states_to_check: if is_job_run_exceeding_expected_runtime( job_run, job_expected_runtime, ): return job_run # check if it is still running at next scheduled job run time if not allow_overlap and queueing and next_run_time: difftime = _timestamp_to_timeobj(next_run_time) if time.time() > time.mktime(difftime): return job_run for action_run in job_run.get("runs", []): if is_action_run_exceeding_expected_runtime( action_run, actions_expected_runtime, ): return job_run next_run_time = job_run.get("run_time", None) return None def is_job_run_exceeding_expected_runtime(job_run, job_expected_runtime): states_to_check = {"running", "waiting", "starting"} if ( job_expected_runtime is not None and job_run.get( "state", "unknown", ) in states_to_check ): duration_seconds = pytimeparse.parse(job_run.get("duration", "")) if duration_seconds and duration_seconds > job_expected_runtime: return True return False def is_action_run_exceeding_expected_runtime( action_run, actions_expected_runtime, ): states_to_check = ["running", "starting"] if action_run.get("state", "unknown") in states_to_check: action_name = action_run.get("action_name", None) if action_name in actions_expected_runtime and actions_expected_runtime[action_name] is not None: duration_seconds = pytimeparse.parse( action_run.get("duration", ""), ) if duration_seconds > actions_expected_runtime[action_name]: return True return False def get_relevant_action(*, action_runs, last_state, actions_expected_runtime): stuck_action_run_candidate = None for action_run in reversed(action_runs): action_state = action_run.get("state", "unknown") try: if State(action_state) == last_state: return action_run except ValueError: if last_state == State.STUCK: if is_action_run_exceeding_expected_runtime( action_run, actions_expected_runtime, ): return action_run if action_state in {"running", "starting"}: stuck_action_run_candidate = action_run return stuck_action_run_candidate or action_runs[-1] def guess_realert_every(job): try: job_next_run = job.get("next_run", None) if job_next_run is None: return -1 job_runs = job.get("runs", []) job_runs_started = [ run.get("start_time") or run.get("run_time") for run in job_runs if run.get("start_time") or run.get("run_time") and run.get("run_time") != job_next_run ] if len(job_runs_started) == 0: return -1 job_previous_run = max( job_runs_started, ) time_diff = time.mktime(_timestamp_to_timeobj(job_next_run)) - time.mktime( _timestamp_to_timeobj(job_previous_run) ) realert_every = max(int(time_diff / _run_interval), 1) except Exception as e: log.warning(f"guess_realert_every failed: {e}") return -1 return realert_every def get_earliest_run_time_to_check(job_content, interval): if not job_content["runs"]: return None earliest_run_time = min(time.mktime(_timestamp_to_timeobj(run["run_time"])) for run in job_content["runs"]) return max( earliest_run_time, time.time() - datetime.timedelta(**{f"{interval}s": NUM_PRECIOUS - 1}).total_seconds(), ) def sort_runs_by_interval(job_content, interval="day", until=None): """Sorts a job's runs by a time interval (day, hour, minute, or second), according to a job run's run time. """ interval_formats = { "day": "%Y.%m.%d", "hour": "%Y.%m.%d-%H", "minute": "%Y.%m.%d-%H.%M", "second": "%Y.%m.%d-%H.%M.%S", } fmt = interval_formats[interval] run_buckets = defaultdict(list) if job_content is not None: if not until: until = time.time() # can't set in default arg earliest_run_time = get_earliest_run_time_to_check(job_content, interval) or until # We add all dates by interval between our earliest run_time and now, # allowing functions downstream to see if some dates had no runs start = datetime.datetime.fromtimestamp(earliest_run_time) end = datetime.datetime.fromtimestamp(until) step = datetime.timedelta(**{f"{interval}s": 1}) # We compare the strings _after_ we've converted to the final format to make # sure we don't miss something due to off-by-one/weird DST bugs, etc while start.strftime(fmt) <= end.strftime(fmt): run_buckets[start.strftime(fmt)] = [] start += step # Bucket runs by interval for run in job_content["runs"]: run_time = time.strftime( interval_formats[interval], _timestamp_to_timeobj(run["run_time"]), ) if run_time not in run_buckets: continue run_buckets[run_time].append(run) return dict(run_buckets) def compute_check_result_for_job(client, job, url_index): kwargs = m( name=f"check_tron_job.{job['name']}", source=client.cluster_name, ) if "realert_every" not in kwargs: kwargs = kwargs.set("realert_every", guess_realert_every(job)) kwargs = kwargs.set("check_every", f"{_run_interval}s") # We want to prevent a monitoring config from setting the check_every # attribute, since one config should not dictate how often this script runs sensu_kwargs = ( pmap(job["monitoring"]) .discard(PRECIOUS_JOB_ATTR) .discard("check_every") .discard("page_for_expected_runtime") .discard("check_oom_events") ) kwargs = kwargs.update(sensu_kwargs) hide_stderr = kwargs.get("hide_stderr", False) kwargs_list = [] if job["status"] == "disabled": kwargs = kwargs.set( "output", f"OK: {job['name']} is disabled and won't be checked.", ) kwargs = kwargs.set("status", 0) kwargs_list.append(kwargs) else: # The job is not disabled, therefore we have to look at its run history tron_id = get_object_type_from_identifier(url_index, job["name"]) job_content = pmap( client.job( tron_id.url, include_action_runs=True, ), ) if job["monitoring"].get(PRECIOUS_JOB_ATTR, False): dated_runs = sort_runs_by_interval(job_content, interval="day") else: dated_runs = {"": job_content["runs"]} for date, runs in dated_runs.items(): results = compute_check_result_for_job_runs( job=job, job_content=job_content.set("runs", runs), client=client, url_index=url_index, hide_stderr=hide_stderr, ) dated_kwargs = kwargs.update(results) if date: # if empty date, leave job name alone dated_kwargs = dated_kwargs.set( "name", f"{kwargs['name']}-{date}", ) kwargs_list.append(dated_kwargs) return [dict(kws) for kws in kwargs_list] def check_job(job, client, url_index): if job.get("monitoring", {}) == {}: log.debug(f"Not checking {job['name']}, no monitoring metadata setup.") return if job.get("monitoring").get("team", None) is None: log.debug(f"Not checking {job['name']}, no team specified") return log.info(f"Checking {job['name']}") return compute_check_result_for_job(job=job, client=client, url_index=url_index) def check_job_result(job, client, url_index, dry_run): results = check_job(job, client, url_index) if not results: return for result in results: if dry_run: log.info("Would have sent this event to sensu: ") log.info(pprint.pformat(result)) else: log.debug(f"Sending event: {pprint.pformat(result)}") if "runbook" not in result: result["runbook"] = ( "No runbook specified. Please specify a runbook in the monitoring section of the job definition.", ) send_event(**result) def main(): args = parse_cli() cmd_utils.setup_logging(args) cmd_utils.load_config(args) client = Client(args.server, args.cluster_name) error_code = 0 global _run_interval _run_interval = args.run_interval global _skip_sensu_failure_logging _skip_sensu_failure_logging = args.skip_sensu_failure_logging url_index = client.index() if args.job is None: jobs = client.jobs(include_job_runs=True) for job in jobs: try: check_job_result(job=job, client=client, url_index=url_index, dry_run=args.dry_run) except Exception as e: log.warning(f"check job result fails for job {job.get('name', '')}: {e}") error_code = 1 else: job_url = client.get_url(args.job) job = client.job_runs(job_url) check_job_result(job=job, client=client, url_index=url_index, dry_run=args.dry_run) return error_code if __name__ == "__main__": sys.exit(main()) ================================================ FILE: tron/bin/get_tron_metrics.py ================================================ #!/usr/bin/env python3.10 # # get_tron_metrics.py # This script is designed to retrieve metrics from Tron via its API and send # send them to meteorite. import logging import pprint import subprocess import sys import textwrap from tron.commands import cmd_utils from tron.commands.client import Client log = logging.getLogger("get_tron_metrics") def parse_cli(): parser = cmd_utils.build_option_parser() parser.description = "Collects metrics from Tron via its API and forwards them to " "meteorite." parser.add_argument( "--dry-run", action="store_true", default=False, help="Don't actually send metrics out. Defaults: %(default)s", ) args = parser.parse_args() return args def check_bin_exists(bin): """ Checks if an executable binary exists :param bin: (str) Name of the executable; could be a path to one """ return ( subprocess.call( ["which", bin], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) == 0 ) def send_data_metric(name, metric_type, value, dimensions={}, dry_run=False): """ Sends a single data point to meteorite via bash command :param name: (str) Name of the metric :param metric_type: (str) Type of the meteorite metric. Must be in METEORITE_TYPES :param value: (float) Value of the metric :param dimensions: (dict) Metric dimensions as key-value pairs :param dry_run: (bool) Whether or not to send metrics to meteorite """ if dry_run: metric_args = dict( name=name, metric_type=metric_type, value=value, dimensions=dimensions, ) log.info( f"Would have sent this to meteorite:\n" f"{pprint.pformat(metric_args)}", ) return cmd = ["meteorite", "data", "-v", name, metric_type, str(value)] for k, v in dimensions.items(): cmd.extend(["-d", f"{k}:{v}"]) process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) output, error = process.communicate() output = output.decode("utf-8").rstrip() error = error.decode("utf-8").rstrip() if process.returncode != 0: log.error( "Meteorite failed with:\n" f"{textwrap.indent(error, ' ')}", ) else: log.debug(f"From meteorite: {output}") def send_counter(name, **kwargs): send_data_metric( name=name, metric_type="counter", value=kwargs.pop("count"), dimensions=kwargs.pop("dimensions", {}), dry_run=kwargs.pop("dry_run", False), ) def send_gauge(name, **kwargs): send_data_metric( name=name, metric_type="gauge", value=kwargs.pop("value"), dimensions=kwargs.pop("dimensions", {}), dry_run=kwargs.pop("dry_run", False), ) def send_meter(name, **kwargs): send_counter(name, **kwargs) # We ignore mX_rate args def send_histogram(name, **kwargs): for k in ["p50", "p75", "p95", "p99"]: # Only send p50-99 gauge_name = f"{name}.{k}" kwargs["value"] = kwargs[k] # set for gauge send_gauge(gauge_name, **kwargs) def send_timer(name, **kwargs): # We mirror the metrics implementation in Tron by splitting timer into a # meter and a histogram send_meter(name, **kwargs) send_histogram(name, **kwargs) _METRIC_SENDERS = { "counter": send_counter, "gauge": send_gauge, "meter": send_meter, "histogram": send_histogram, "timer": send_timer, } def send_metrics(metrics, cluster=None, dry_run=False): """ Send metrics via meteorite :param metrics: Dictionary of metrics types and their data """ for metric_type, data in metrics.items(): for kwargs in data: name = kwargs.pop("name") kwargs["dry_run"] = dry_run if cluster: dimensions = kwargs.get("dimensions", {}) dimensions["tron_cluster"] = cluster kwargs["dimensions"] = dimensions _METRIC_SENDERS[metric_type](name, **kwargs) def main(): args = parse_cli() cmd_utils.setup_logging(args) cmd_utils.load_config(args) client = Client(args.server, args.cluster_name) if check_bin_exists("meteorite"): metrics = client.metrics() send_metrics(metrics, cluster=client.cluster_name, dry_run=args.dry_run) else: log.error("'meteorite' was not found") if __name__ == "__main__": sys.exit(main()) ================================================ FILE: tron/bin/recover_batch.py ================================================ #!/usr/bin/env python3.10 import argparse import logging import signal import sys from queue import Queue import psutil from twisted.internet import inotify from twisted.internet import reactor from twisted.python import filepath from tron import yaml log = logging.getLogger("tron.recover_batch") class StatusFileWatcher: """ Watches the status file produced by action runners """ def __init__(self, to_watch, callback): notifier = inotify.INotify() notifier.startReading() notifier.watch(filepath.FilePath(to_watch), callbacks=[callback]) def parse_args(): parser = argparse.ArgumentParser( description="Check if a action runner has exited; wait otherwise", ) parser.add_argument("filepath") return parser.parse_args() def read_last_yaml_entries(filename): with open(filename) as f: lines = list(yaml.load_all(f)) if not lines: entries = {} else: entries = lines[-1] return entries def notify(notify_queue, ignored, filepath, mask): exit_code, error_message = get_exit_code(filepath.path) if exit_code is not None: reactor.stop() notify_queue.put((exit_code, error_message)) def get_exit_code(filepath): entries = read_last_yaml_entries(filepath) pid = entries.get("runner_pid") return_code = entries.get("return_code") exit_code, error_message = None, None if return_code is not None: if return_code < 0: # from the subprocess docs on the return code of a process: # "A negative value -N indicates that the child was terminated by signal N (POSIX only)." # We should always exit with a positive code, so we take the absolute value of the return code exit_code = abs(return_code) error_message = f"Action run killed by signal {signal.Signals(exit_code).name}" else: exit_code = return_code elif pid is None: log.warning(f"Status file {filepath} didn't have a PID. Will watch the file for updates.") elif not psutil.pid_exists(pid): exit_code = 1 error_message = f"Action runner pid {pid} no longer running. Assuming an exit of 1." return exit_code, error_message def run(fpath): # Check if the process has already completed. # If it has, we don't expect any more updates. return_code, error_message = get_exit_code(fpath) if return_code is not None: if error_message is not None: log.warning(error_message) sys.exit(return_code) # If not, wait for updates to the file. notify_queue = Queue() StatusFileWatcher( fpath, lambda *args, **kwargs: notify(notify_queue, *args, **kwargs), ) reactor.run() exit_code, error_message = notify_queue.get() if error_message is not None: log.warning(error_message) sys.exit(exit_code) if __name__ == "__main__": args = parse_args() run(args.filepath) ================================================ FILE: tron/command_context.py ================================================ """Command Context is how we construct the command line for a command which may have variables that need to be rendered. """ import operator import re from functools import reduce from tron.utils import timeutils def build_context(object, parent): """Construct a CommandContext for object. object must have a property 'context_class'. """ return CommandContext(object.context_class(object), parent) def build_filled_context(*context_objects): """Create a CommandContext chain from context_objects, using a Filler object to pass to each CommandContext. Can be used to validate a format string. """ if not context_objects: return CommandContext() filler = Filler() def build(current, next): return CommandContext(next(filler), current) return reduce(build, context_objects, None) class CommandContext: """A CommandContext object is a wrapper around any object which has values to be used to render a command for execution. It looks up values by name. It's lookup order is: base[name], base.__getattr__(name), next[name], next.__getattr__(name) """ def __init__(self, base=None, next=None): """ base - Object to look for attributes in next - Next place to look for more pieces of context Generally this will be another instance of CommandContext """ self.base = base or {} self.next = next or {} def get(self, name, default=None): try: return self.__getitem__(name) except KeyError: return default def __getitem__(self, name): getters = [operator.itemgetter(name), operator.attrgetter(name)] for target in [self.base, self.next]: for getter in getters: try: return getter(target) except (KeyError, TypeError, AttributeError): pass raise KeyError(name) def __eq__(self, other): return self.base == other.base and self.next == other.next def __ne__(self, other): return not self == other class JobContext: """A class which exposes properties for rendering commands.""" def __init__(self, job): self.job = job @property def name(self): return self.job.name def __getitem__(self, item): date_name, date_spec = self._get_date_spec_parts(item) if not date_spec: raise KeyError(item) if date_name == "last_success": last_success = self.job.runs.last_success last_success = last_success.run_time if last_success else None time_value = timeutils.DateArithmetic.parse( date_spec, last_success, ) if time_value: return time_value raise KeyError(item) def _get_date_spec_parts(self, name): parts = name.rsplit("#", 1) if len(parts) != 2: return name, None return parts @property def namespace(self): return self.job.name.split(".")[0] class JobRunContext: def __init__(self, job_run): self.job_run = job_run @property def runid(self): return self.job_run.id @property def manual(self): return str(self.job_run.manual).lower() @property def cleanup_job_status(self): """Provide 'SUCCESS' or 'FAILURE' to a cleanup action context based on the status of the other steps """ if self.job_run.action_runs.is_failed: return "FAILURE" elif self.job_run.action_runs.is_complete_without_cleanup: return "SUCCESS" return "UNKNOWN" def __getitem__(self, name): """ This function attempts to parse any command context variable expressions that use shortdate or runid in the following order: 1) Attempt to parse date arithmetic syntax and apply to run_time unconditionally and, if unsuccessful falls to the next case 2) Attempts to parse a delta to apply to the current job runid - this is mostly meant to be used for jobs that rely on the output of the previous run, but this is not enforced in case someone can dream up another scenario where they want to do arbitrary deltas here. """ run_time = self.job_run.run_time time_value = timeutils.DateArithmetic.parse(name, run_time) if time_value: return time_value # this is a little weird, but enumerating the cases that should be parsed by timeutils is hard, # so we just unconditionally attempt to parse the name and then fallback to the runid special cases # rather than attempt to enumerate the timeutils cases elif name == "runid": # we could expand the logic below to handle this with the regex, but that # would make the code a little more complex for not much gain return self.runid elif "runid" in name: # we're really only expecting runid-1 for now but, as described in the docstring, # we're allowing arbitrary addition/subtration in case someone dreams up a use for # them match = re.match(r"^runid([+-]\d+)$", name) if match: # self.runid here will be the job runid (e.g., NAMESPACE.SERVICE.RUN_NUMBER) - it will not # include an action name. # that said - all we need math-wise here is the run number, so we split on . and store the job name # so that we can re-consistitute the runid after doing math on the run number job_name, run_num = self.runid.rsplit(".", maxsplit=1) # NOTE: this will potentially return a runid for a job that will never exist - e.g., if you setup an # action that should only run after the previous jobrun's action has run for a job that has never run # before) - normally this will only be a problem for the very first run and users can easily tronctl start # the action to bootstrap things so we don't do any checking to see if the returned runid is valid return f"{job_name}.{int(run_num) + int(match.groups()[0])}" raise KeyError(name) class ActionRunContext: """Context object that gives us access to data about the action run.""" def __init__(self, action_run): self.action_run = action_run @property def actionname(self): return self.action_run.action_name @property def node(self): return self.action_run.node.hostname class Filler: """Filler object for using CommandContext during config parsing. This class is used as a substitute for objects that would be passed to Context objects. This allows the Context objects to be used directly for config validation. """ def __getattr__(self, _): return self def __str__(self): return "%(...)s" def __mod__(self, _): return self def __nonzero__(self): return False def __bool__(self): return False ================================================ FILE: tron/commands/__init__.py ================================================ ================================================ FILE: tron/commands/authentication.py ================================================ import os from typing import cast from tron.commands.cmd_utils import get_client_config try: from vault_tools.oidc import get_instance_oidc_identity_token # type: ignore # library lacks py.typed marker from okta_auth import get_and_cache_jwt_default # type: ignore # library lacks py.typed marker except ImportError: def get_instance_oidc_identity_token(role: str, ecosystem: str | None = None) -> str: return "" def get_and_cache_jwt_default(client_id: str, refreshable: bool = False, force: bool = False) -> str: return "" def get_sso_auth_token(no_cache: bool = False) -> str: """Generate an authentication token for the calling user from the Single Sign On provider, if configured""" client_id = get_client_config().get("auth_sso_oidc_client_id") return cast(str, get_and_cache_jwt_default(client_id, refreshable=True, force=no_cache)) if client_id else "" def get_vault_auth_token() -> str: """Generate an authentication token for the underlying instance via Vault""" vault_role = get_client_config().get("vault_api_auth_role", "service_authz") return cast(str, get_instance_oidc_identity_token(vault_role)) def get_auth_token(no_cache: bool = False) -> str: """Generate authentication token via Vault or Okta""" return get_vault_auth_token() if os.getenv("TRONCTL_VAULT_AUTH") else get_sso_auth_token(no_cache) ================================================ FILE: tron/commands/backfill.py ================================================ import asyncio import datetime import functools import os import pprint import re import signal import sys from urllib.parse import urljoin from tron.commands import client from tron.commands import display from tron.commands.authentication import get_auth_token from tron.core.actionrun import ActionRun DEFAULT_MAX_PARALLEL_RUNS = 3 LIMIT_MAX_PARALLEL_RUNS = 10 DEFAULT_POLLING_INTERVAL_S = 10 def get_date_range( start_date: datetime.datetime, end_date: datetime.datetime, descending: bool = False, ) -> list[datetime.datetime]: dates = [] delta = end_date - start_date for days_to_add in range(delta.days + 1): dates.append(start_date + datetime.timedelta(days=days_to_add)) if descending: dates.reverse() return dates def print_backfill_cmds(job: str, date_strs: list[str]) -> None: print(f"Please run the following {len(date_strs)} commands:") print("") for date in date_strs: print(f"tronctl start {job} --run-date {date}") print("") print("Note that many jobs operate on the previous day's data.") def confirm_backfill(job: str, date_strs: list[str]) -> bool: print( f"To backfill for the job '{job}', a job run will be created for each " f"of the following {len(date_strs)} dates:" ) pprint.pprint(date_strs) print("") user_resp = input("Confirm? [y/n] ") if user_resp.lower() != "y": print("Aborted.") return False else: print("") # just for clean separation return True class BackfillRun: NOT_STARTED_STATE = "not started" SUCCESS_STATES = {ActionRun.SUCCEEDED, ActionRun.CANCELLED, ActionRun.SKIPPED} MAX_SYNC_FAILURES = 5 def __init__(self, tron_client: client.Client, job_id: client.TronObjectIdentifier, run_time: datetime.datetime): self.tron_client = tron_client self.job_id = job_id self.run_time = run_time self.run_name: str | None = None self.run_id: client.TronObjectIdentifier | None = None self.run_state = BackfillRun.NOT_STARTED_STATE @property def run_time_str(self) -> str: return self.run_time.date().isoformat() async def run_until_completion(self) -> str: """Runs this job run until it finishes (i.e. reaches a terminal state).""" try: if await self.create(): await self.sync_state() await self.watch_until_completion() except asyncio.CancelledError: await self.cancel() return self.run_state async def create(self) -> str | None: """Creates job run for a specific date. Returns the name of the run, if it was created with no issues. """ # create the job run loop = asyncio.get_event_loop() response = await loop.run_in_executor( None, functools.partial( client.request, urljoin(self.tron_client.url_base, self.job_id.url), data=dict(command="start", run_time=self.run_time), user_attribution=True, ), ) # figure out its name if response.error: print(f"Error: couldn't start job run for {self.run_time_str}: {response.content}") else: # determine name of job run so that we can watch it # from tron.api.controller.JobController and tron.core.jobrun, the format # of the response result will be: "Created JobRun:" result = response.content.get("result") match = re.match(r"^Created JobRun:([-.\w]+)$", result) if match: self.run_name = match.groups(0)[0] # type: ignore[assignment] # mypy wrongly identifies self.run_name type as "Union[str, int]" self.run_state = ActionRun.STARTING print(f"Job run '{self.run_name}' for {self.run_time_str} created") else: print( f"Warning: Job run for {self.run_time_str} created, but couldn't determine " "its name, so its state is considered to be unknown." ) self.run_state = ActionRun.UNKNOWN return self.run_name async def get_run_id(self) -> client.TronObjectIdentifier | None: if not self.run_id: loop = asyncio.get_event_loop() try: self.run_id = await loop.run_in_executor( None, client.get_object_type_from_identifier, self.tron_client.index(), self.run_name, ) except client.RequestError as e: print(f"Error: couldn't get resource URL for job run '{self.run_name}': {e}") return self.run_id async def sync_state(self) -> str: """Syncs the local run state with that of the Tron server's. Returns the updated state. """ if not self.run_id: self.run_id = await self.get_run_id() if self.run_id: loop = asyncio.get_event_loop() try: # get the state of the run using the resource url resp_content = await loop.run_in_executor( None, functools.partial( self.tron_client.job_runs, urljoin(self.tron_client.url_base, self.run_id.url), include_runs=False, include_graph=False, ), ) self.run_state = resp_content.get("state", ActionRun.UNKNOWN) except (client.RequestError, AttributeError, ValueError) as e: print(f"Error: couldn't get state for job run '{self.run_name}': {e}") self.run_state = ActionRun.UNKNOWN return self.run_state async def watch_until_completion(self, poll_intv_s: int = DEFAULT_POLLING_INTERVAL_S) -> str: """Watches this job run until it finishes. Returns the end state of the run. """ sync_failures = 0 while self.run_state not in ActionRun.END_STATES: await asyncio.sleep(poll_intv_s) try: await self.sync_state() sync_failures = 0 except Exception as e: print(f"Issue syncing state for '{self.run_name}': {e}", file=sys.stderr) sync_failures += 1 if sync_failures > self.MAX_SYNC_FAILURES: raise print(f"Job run '{self.run_name}' for {self.run_time_str} finished with state: {self.run_state}") return self.run_state async def cancel(self) -> bool: """Cancel this run if it is running. Returns whether or not the run was successfully cancelled """ if self.run_id: loop = asyncio.get_event_loop() response = await loop.run_in_executor( None, functools.partial( client.request, urljoin(self.tron_client.url_base, self.run_id.url), data=dict(command="cancel"), user_attribution=True, ), ) if response.error: print( f"Error: couldn't cancel '{self.run_name}' for {self.run_time_str}. " "You should use tronview to check on it." ) else: print(f"Backfill job run '{self.run_name}' for {self.run_time_str} cancelled") self.run_state = ActionRun.CANCELLED return True else: # accounts for the case where the job was created, but this coroutine # is cancelled before the name (and id) is returned to us print( f"Warning: attempted to cancel backfill for {self.run_time_str}, but we " "don't know if it was created initially. You should use tronview " "to check." ) return False async def run_backfill_for_date_range( server: str, job_name: str, dates: list[datetime.datetime], max_parallel: int = DEFAULT_MAX_PARALLEL_RUNS, ignore_errors: bool = True, ) -> list[BackfillRun]: """Creates and watches job runs over a range of dates for a given job. At most, max_parallel runs can run in parallel to prevent resource exhaustion. """ loop = asyncio.get_event_loop() # Trigger authentication before submitting all async jobs, so auth tokens # are cached and won't prompt the user in the individual API calls. # We pass `no_cache` to ensure a new refresh token is generated and stored in memory. if os.getenv("TRONCTL_API_AUTH"): get_auth_token(no_cache=True) tron_client = client.Client(server, user_attribution=True) url_index = tron_client.index() # check job_name identifies a valid tron object job_id = client.get_object_type_from_identifier(url_index, job_name) # check job_name identifies a job if job_id.type != client.TronObjectType.job: raise ValueError(f"'{job_name}' is a {job_id.type.lower()}, not a job") backfill_runs = [BackfillRun(tron_client, job_id, run_time) for run_time in dates] running: set[asyncio.Future] = set() finished_cnt = 0 all_successful = True # `current_task()` will always return a task here, but we need to account # for the None case for mypy current_task = asyncio.current_task() if current_task: loop.add_signal_handler(signal.SIGINT, current_task.cancel) try: while finished_cnt < len(dates): # start more runs if we still have some and tha parallel limit is not yet reached while finished_cnt + len(running) < len(dates) and len(running) < max_parallel: next_run = backfill_runs[finished_cnt + len(running)] running.add(asyncio.ensure_future(next_run.run_until_completion())) just_finished, running = await asyncio.wait(running, return_when=asyncio.FIRST_COMPLETED) for task in just_finished: finished_cnt += 1 all_successful &= task.result() in BackfillRun.SUCCESS_STATES if not ignore_errors and not all_successful: print("Error: encountered failing job run; cancelling all in-progress runs and exiting.") for task in running: task.cancel() # cancel running async tasks await task # wait until it is done cancelling break except asyncio.CancelledError: # caused by sigint handler print("Error: SIGINT detected; aborting all in-progress runs and exiting") for task in running: task.cancel() await task return backfill_runs class DisplayBackfillRuns(display.TableDisplay): columns = ["Date", "Job Run Name", "Final State"] fields = ["run_time", "run_name", "run_state"] widths = [15, 60, 15] title = "Backfills Job Runs" resize_fields = {"run_time", "run_name", "run_state"} header_color = "hgray" def print_backfill_runs_table(runs: list[BackfillRun]) -> None: """Prints backfill runs in a table""" with display.Color.enable(): table = DisplayBackfillRuns().format( [ dict(run_time=r.run_time.date().isoformat(), run_name=(r.run_name or "n/a"), run_state=r.run_state) for r in runs ] ) print(table) ================================================ FILE: tron/commands/client.py ================================================ """ A command line http client used by tronview, tronctl, and tronfig """ import json import logging import os import urllib.error import urllib.parse import urllib.request from collections import namedtuple import tron from tron.commands.authentication import get_auth_token from tron.config.schema import MASTER_NAMESPACE log = logging.getLogger(__name__) USER_AGENT = f"Tron Command/{tron.__version__} +http://github.com/Yelp/Tron" DECODE_ERROR = "DECODE_ERROR" URL_ERROR = "URL_ERROR" class RequestError(ValueError): """Raised when the request to tron API fails.""" Response = namedtuple("Response", "error msg content") default_headers = { "User-Agent": USER_AGENT, } def build_url_request(uri, data, headers=None, method=None): headers = headers or default_headers enc_data = urllib.parse.urlencode(data).encode() if data else None # Currently implementing auth only for management actions (i.e. POST requests) if os.getenv("TRONCTL_API_AUTH") and (data or (method and method.upper() == "POST")): token = get_auth_token() if token: headers["Authorization"] = f"Bearer {token}" return urllib.request.Request(uri, enc_data, headers=headers, method=method) def load_response_content(http_response): encoding = http_response.headers.get_content_charset() if encoding is None: encoding = "utf8" content = http_response.read().decode(encoding) try: return Response(None, None, json.loads(content)) except ValueError as e: log.error("Failed to decode response: %s, %s", e, content) return Response(DECODE_ERROR, str(e), content) def build_http_error_response(exc): content = exc.read() if hasattr(exc, "read") else None if content: encoding = exc.headers.get_content_charset() if encoding is None: encoding = "utf8" content = content.decode(encoding) try: content = json.loads(content) content = content["error"] except ValueError: log.warning( f"Incorrectly formatted error response: {content}", ) return Response(exc.code, exc.msg, content) def request(uri, data=None, headers=None, method=None, user_attribution=False): log.info("Request to %s with %s", uri, data) headers = headers or default_headers if user_attribution: headers = ensure_user_attribution(headers) request = build_url_request(uri, data, headers=headers, method=method) try: response = urllib.request.urlopen(request) except urllib.error.HTTPError as e: log.error("Received error response: %s" % e) return build_http_error_response(e) except urllib.error.URLError as e: log.error("Received error response: %s" % e) return Response(URL_ERROR, e.reason, None) return load_response_content(response) def build_get_url(url, data=None): if data: query_str = urllib.parse.urlencode(sorted(data.items())) return f"{url}?{query_str}" else: return url def ensure_user_attribution(headers: dict[str, str]) -> dict[str, str]: headers = headers.copy() if "User-Agent" not in headers: headers["User-Agent"] = USER_AGENT headers["User-Agent"] += f' ({os.environ.get("USER", "anonymous")})' return headers class Client: """An HTTP client used to issue commands to the Tron API.""" def __init__(self, url_base, cluster_name=None, user_attribution=False): """Create a new client. url_base - A url with a schema, hostname and port """ self.url_base = url_base self.cluster_name = cluster_name self.headers = default_headers if user_attribution: self.headers = ensure_user_attribution(self.headers) def status(self): return self.http_get("/api/status") def metrics(self): return self.http_get("/api/metrics") def config( self, config_name, config_data=None, config_hash=None, check=False, ): """Retrieve or update the configuration.""" if config_data is not None: data_check = 1 if check else 0 request_data = dict( config=config_data, name=config_name, hash=config_hash, check=data_check, ) return self.request("/api/config", request_data) request_data = dict(name=config_name) return self.http_get("/api/config", request_data) def home(self): return self.http_get("/api/") index = home def get_url(self, identifier): return get_object_type_from_identifier(self.index(), identifier).url def jobs( self, include_job_runs=False, include_action_runs=False, include_action_graph=True, include_node_pool=True, ): params = { "include_job_runs": int(include_job_runs), "include_action_runs": int(include_action_runs), "include_action_graph": int(include_action_graph), "include_node_pool": int(include_node_pool), } return self.http_get("/api/jobs", params).get("jobs") def job(self, job_url, include_action_runs=False, count=0): params = { "include_action_runs": int(include_action_runs), "num_runs": count, } return self.http_get(job_url, params) def job_runs(self, url, include_runs=True, include_graph=False): params = { "include_action_runs": int(include_runs), "include_action_graph": int(include_graph), } return self.http_get(url, params) def action_runs(self, action_run_url, num_lines=0): params = { "num_lines": num_lines, "include_stdout": 1, "include_stderr": 1, } return self.http_get(action_run_url, params) def http_get(self, url, data=None): return self.request(build_get_url(url, data)) def request(self, url, data=None): uri = urllib.parse.urljoin(self.url_base, url) response = request(uri, data, headers=self.headers) if response.error: if response.content: raise RequestError(response.content) else: raise RequestError(f"{response.error} {response.msg}") return response.content def build_api_url(resource, identifier_parts): return "/api/{}/{}".format(resource, "/".join(identifier_parts)) def split_identifier(identifier): return identifier.rsplit(".", identifier.count(".") - 1) def get_job_url(identifier): return build_api_url("jobs", split_identifier(identifier)) class TronObjectType: """Constants to identify a Tron object type.""" job = "JOB" job_run = "JOB_RUN" action_run = "ACTION_RUN" url_builders = { "jobs": get_job_url, } groups = { "jobs": [job, job_run, action_run], } TronObjectIdentifier = namedtuple("TronObjectIdentifier", "type url") IdentifierParts = namedtuple("IdentifierParts", "name full_id length") def first(seq): for item in filter(None, seq): return item def get_object_type_from_identifier(url_index, identifier): """Given a string identifier, return a TronObjectIdentifier.""" name_mapping = { "jobs": set(url_index["jobs"]), } def get_name_parts(identifier, namespace=None): if namespace: identifier = f"{namespace}.{identifier}" name_elements = identifier.split(".") name = ".".join(name_elements[:2]) length = len(name_elements) - 2 return IdentifierParts(name, identifier, length) def find_by_type(id_parts, index_name): url_type_index = name_mapping[index_name] if id_parts.name in url_type_index: tron_type = TronObjectType.groups[index_name][id_parts.length] url = TronObjectType.url_builders[index_name](id_parts.full_id) return TronObjectIdentifier(tron_type, url) def find_by_name(name, namespace=None): id = get_name_parts(name, namespace) return find_by_type(id, "jobs") namespaces = [None, MASTER_NAMESPACE] + url_index["namespaces"] id_obj = first(find_by_name(identifier, name) for name in namespaces) if id_obj: return id_obj raise ValueError("Unknown job identifier: %s" % identifier) ================================================ FILE: tron/commands/cmd_utils.py ================================================ """ Common code for command line utilities (see bin/) """ import argparse import difflib import logging import os import sys import tron from tron import yaml log = logging.getLogger("tron.commands") class ExitCode: """Enumeration of exit status codes.""" success = 0 fail = 1 GLOBAL_CONFIG_FILE_NAME = ( os.environ.get( "TRON_CONFIG", ) or "/etc/tron/tron.yaml" ) CONFIG_FILE_NAME = os.path.expanduser("~/.tron") DEFAULT_HOST = "localhost" DEFAULT_PORT = 8089 DEFAULT_CONFIG = { "server": "http://%s:%d" % (DEFAULT_HOST, DEFAULT_PORT), "display_color": False, "cluster_name": "Unnamed Cluster", } TAB_COMPLETE_FILE = "/var/cache/tron_tab_completions" COLOR_RED = "\033[31m" COLOR_YELLOW = "\033[33m" COLOR_DEFAULT = "\033[0m" opener = open def get_default_server(): return DEFAULT_CONFIG["server"] def filter_jobs_actions_runs(prefix, inputs): dots = prefix.count(".") if prefix == "": # If the user hasn't begun to type anything, we need to get them started with all jobs return [i for i in inputs if i.count(".") == 1] elif dots == 0: # If the user hasn't completed a job, we need to get them started with all jobs # that start with what they have return [i for i in inputs if i.count(".") == 1 and i.startswith(prefix)] elif prefix in inputs: # If what a user typed is exactly what is already in a suggestion, then we need to give them # Even more suggestions (+1) return [i for i in inputs if i.startswith(prefix) and (i.count(".") == dots or i.count(".") == dots + 1)] else: # Otherwise we only want to scope our suggestions to those that are on the same "level" # which in string form means they have the same number of dots return [i for i in inputs if i.startswith(prefix) and i.count(".") == dots] def tron_jobs_completer(prefix, **kwargs): if os.path.isfile(TAB_COMPLETE_FILE): with opener(TAB_COMPLETE_FILE, "r") as f: jobs = f.readlines() return filter_jobs_actions_runs( prefix=prefix, inputs=[job.strip("\n\r") for job in jobs], ) else: # this import is here to avoid an annoying circular dependency from tron.commands.client import Client if "client" not in kwargs: client = Client(get_default_server()) else: client = kwargs["client"] return filter_jobs_actions_runs( prefix=prefix, inputs=[job["name"] for job in client.jobs()], ) def build_option_parser(usage=None, epilog=None): parser = argparse.ArgumentParser( usage=usage, epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--version", action="version", version=f"{parser.prog} {tron.__version__}", ) parser.add_argument( "-v", "--verbose", action="count", help="Verbose logging", default=None, ) parser.add_argument( "--server", default=None, help="Url including scheme, host and port, Default: %(default)s", ) parser.add_argument( "--cluster_name", default=None, help="Human friendly tron cluster name", ) parser.add_argument( "-s", "--save", action="store_true", dest="save_config", help="Save options used on this job for next time.", ) return parser def get_client_config(): config_file_list = [CONFIG_FILE_NAME, GLOBAL_CONFIG_FILE_NAME] for config_file in config_file_list: filename = os.path.expanduser(config_file) if os.access(filename, os.R_OK): config = read_config(filename) if config: return config log.debug("Could not find a config in: %s." % ", ".join(config_file_list)) return {} def load_config(options): """Attempt to load a user specific configuration or a global config file and set any unset options based on values from the config. Finally fallback to DEFAULT_CONFIG for those settings. Also save back options to the config if options.save_config is True. """ config = get_client_config() for opt_name in DEFAULT_CONFIG.keys(): if not hasattr(options, opt_name): continue if getattr(options, opt_name) is not None: continue default_value = DEFAULT_CONFIG[opt_name] setattr(options, opt_name, config.get(opt_name, default_value)) if options.save_config: save_config(options) def read_config(filename=CONFIG_FILE_NAME): try: with opener(filename, "r") as config_file: return yaml.load(config_file) except OSError: log.info("Failed to read config file: %s" % CONFIG_FILE_NAME) return {} def write_config(config): with open(CONFIG_FILE_NAME, "w") as config_file: yaml.dump(config, config_file) def save_config(options): config = read_config() for opt_name in DEFAULT_CONFIG.keys(): if not hasattr(options, opt_name): continue config[opt_name] = getattr(options, opt_name) write_config(config) def setup_logging(options: argparse.Namespace) -> int: if options.verbose is None: level = logging.ERROR elif options.verbose == 1: level = logging.WARNING elif options.verbose == 2: level = logging.INFO else: level = logging.NOTSET logging.basicConfig( level=level, format="%(name)s %(levelname)s %(message)s", stream=sys.stdout, ) return level def suggest_possibilities(word, possibilities, max_suggestions=6): suggestions = difflib.get_close_matches( word=word, possibilities=possibilities, n=max_suggestions, ) if len(suggestions) == 1: return f"\nDid you mean: {suggestions[0]}?" elif len(suggestions) >= 1: return f"\nDid you mean one of: {', '.join(suggestions)}?" else: return "" def warning_output(text: str, color: str = COLOR_RED) -> str: """Return the passed-in string colored in red (by default). Suitable for warning messages.""" return f"{color}{text}{COLOR_DEFAULT}" ================================================ FILE: tron/commands/display.py ================================================ """ Format and color output for tron commands. """ import contextlib from collections.abc import Callable from collections.abc import Collection from functools import partial from operator import itemgetter from tron.core import actionrun from tron.core import job from tron.utils import exitcode from tron.utils import maybe_encode class Color: enabled = None colors = { "gray": "\033[90m", "red": "\033[91m", "green": "\033[92m", "yellow": "\033[93m", "blue": "\033[94m", "purple": "\033[95m", "cyan": "\033[96m", "white": "\033[99m", # h is for highlighted "hgray": "\033[100m", "hred": "\033[101m", "hgreen": "\033[102m", "hyellow": "\033[103m", "hblue": "\033[104m", "hcyan": "\033[106m", "end": "\033[0m", } @classmethod @contextlib.contextmanager def enable(cls): old_val = cls.enabled try: cls.enabled = True yield finally: cls.enabled = old_val @classmethod def set(cls, color_name, text): if not cls.enabled or not color_name: return text return "{}{}{}".format( cls.colors[color_name.lower()], text, cls.colors["end"], ) @classmethod def toggle(cls, enable): cls.enabled = enable class TableDisplay: """Base class for displaying columns of data. This class takes a list of dict objects and formats it so that it displays properly in fixed width columns. Overlap is truncated. This class provides many hooks for customizing the output, including: - sorting of rows - building composite values from more then one field - custom formatting of a columns values - adding additional data after each row - coloring of header, columns, or rows The default output is: Banner Header Row (optional post row) Row (optional post row) ... Footer """ columns: list[str] = [] fields: list[str] = [] widths: list[int] = [] colors: dict[str, Callable[[str], str]] = {} title: str | None = None resize_fields: Collection[str] = set() reversed = False header_color = "hgray" def __init__(self, sort_index=0): self.out = [] self.sort_index = sort_index def banner(self): if not self.title: return title = self.title.capitalize() self.out.append("\n%s:" % title) if not self.rows(): self.out.append("No %s\n" % title) def header(self): row = [label.ljust(self.get_field_width(i)) for i, label in enumerate(self.columns)] self.out.append(Color.set(self.header_color, "".join(row))) def footer(self): pass def color(self, col, field): return None def sorted_fields(self, values): return [values[name] for name in self.fields] def format_row(self, fields): row = [ Color.set(self.color(i, value), self.trim_value(i, value)) for i, value in enumerate(self.sorted_fields(fields)) ] return Color.set(self.row_color(fields), "".join(row)) def get_field_width(self, field_idx): return self.widths[field_idx] def trim_value(self, field_idx, value): length = self.get_field_width(field_idx) value = self.format_value(field_idx, value) if len(value) > length: return (value[: length - 3] + "...").ljust(length) return value.ljust(length) def format_value(self, field_idx, value): return str(value) def output(self): out = "\n".join(self.out) self.out = [] return out def post_row(self, row): pass def row_color(self, row): return None def rows(self): return sorted( self.data, key=itemgetter(self.fields[self.sort_index]), reverse=self.reversed, ) def store_data(self, data): self.data = data def update_column_widths(self): """Update column widths to fit the data.""" for field_idx, field in enumerate(self.fields): if field in self.resize_fields: self.widths[field_idx] = self.calculate_width(field_idx) def calculate_width(self, field_idx): default_width = self.widths[field_idx] column = [self.format_value(field_idx, row[self.fields[field_idx]]) for row in self.data] if not column: return default_width max_value_width = max(len(value) for value in column) return max(max_value_width + 1, default_width) def format(self, data): self.store_data(data) self.update_column_widths() self.banner() if not self.rows(): return self.output() self.header() for row in self.rows(): self.out.append(self.format_row(row)) self.post_row(row) self.footer() return self.output() def add_color_for_state(state): if state == actionrun.ActionRun.FAILED: return Color.set("red", state) if state in { actionrun.ActionRun.RUNNING, actionrun.ActionRun.SUCCEEDED, job.Job.STATUS_ENABLED, }: return Color.set("green", state) if state in {job.Job.STATUS_DISABLED}: return Color.set("blue", state) return state def format_fields(display_obj, content): """Format fields with some color.""" def add_color(field, field_value): if field not in display_obj.colors: return field_value return display_obj.colors[field](field_value) def format_field(field): formatter = field_display_mapping.get(field, lambda f, _: f) return formatter(content.get(field), content) def build_field(label, field): return f"{label:<20}: {add_color(field, format_field(field))}" return "\n".join(build_field(*item) for item in display_obj.detail_labels) def format_job_details(job_content): details = format_fields(DisplayJobs, job_content) job_runs = DisplayJobRuns().format(job_content["runs"]) actions = "\n\nList of Actions:\n%s" % "\n".join( job_content["action_names"], ) return details + actions + "\n" + job_runs def format_action_run_details(content, stdout=True, stderr=True): out = ["Requirements:"] + content["requirements"] + [""] if stdout: out.append("Stdout:\n%s\n" % "\n".join(content["stdout"])) if stderr: out.append("Stderr:\n%s\n" % "\n".join(content["stderr"])) details = format_fields(DisplayActionRuns, content) return details + "\n" + "\n".join(out) class DisplayJobRuns(TableDisplay): """Format Job runs.""" columns = ["Run ID", "State", "Node", "Scheduled Time"] fields = ["run_num", "state", "node", "run_time"] widths = [10, 12, 30, 25] title = "job runs" reversed = True detail_labels = [ ("Job Run", "id"), ("State", "state"), ("Node", "node"), ("Scheduled time", "run_time"), ("Start time", "start_time"), ("End time", "end_time"), ("Manual run", "manual"), ] colors = { "id": partial(Color.set, "yellow"), "state": add_color_for_state, "manual": lambda value: Color.set("cyan" if value else None, value), } def format_value(self, field_idx, value): if self.fields[field_idx] == "run_num": value = "." + str(value) if self.fields[field_idx] == "scheduled_time": value = value or "-" if self.fields[field_idx] == "node": value = display_node(value) return super().format_value(field_idx, value) def row_color(self, fields): return "red" if fields["state"] == "FAIL" else "white" def post_row(self, row): start = row["start_time"] or "-" end = row["end_time"] or "-" duration = row["duration"][:-7] if row["duration"] else "-" row_data = "{}Start: {} End: {} ({})".format( " " * self.widths[0], start, end, duration, ) self.out.append(Color.set("gray", row_data)) class DisplayJobs(TableDisplay): columns = ["Name", "State", "Scheduler", "Last Success"] fields = ["name", "status", "scheduler", "last_success"] widths = [50, 10, 20, 22] title = "jobs" resize_fields = ["name"] detail_labels = [ ("Job", "name"), ("State", "status"), ("Scheduler", "scheduler"), ("Max runtime", "max_runtime"), ("Node Pool", "node_pool"), ("Run on all nodes", "all_nodes"), ("Allow overlapping", "allow_overlap"), ("Queue overlapping", "queueing"), ] colors = { "name": partial(Color.set, "yellow"), "status": add_color_for_state, } def format_value(self, field_idx, value): if self.fields[field_idx] == "scheduler": value = display_scheduler(value) return super().format_value(field_idx, value) class DisplayActionRuns(TableDisplay): columns = ["Action", "State", "Start Time", "End Time", "Duration"] fields = ["id", "state", "start_time", "end_time", "duration"] widths = [40, 12, 22, 22, 10] title = "actions" resize_fields = ["id"] detail_labels = [ ("Action Run", "id"), ("State", "state_delayed"), ("Node", "node"), ("Last run command", "command"), ("Original raw command", "original_command"), ("Config command", "raw_command"), ("Start time", "start_time"), ("End time", "end_time"), ("Final exit status", "exit_status"), ("Exit statuses", "exit_statuses"), ("Waits for triggers", "triggered_by"), ("Publishes triggers", "trigger_downstreams"), ] colors = { "id": partial(Color.set, "yellow"), "state": add_color_for_state, "command": partial(Color.set, "gray"), } def __init__(self): # Action runs need to be storted by start time, which is index 2 super().__init__(sort_index=2) def banner(self): self.out.append(format_fields(DisplayJobRuns, self.job_run)) super().banner() def format_value(self, field_idx, value): if self.fields[field_idx] == "id": value = "." + value.rsplit(".", 1)[-1] if self.fields[field_idx] in ("start_time", "end_time"): value = value or "-" if self.fields[field_idx] == "duration": # Strip microseconds value = value[:-7] if value else "-" return super().format_value(field_idx, value) def row_color(self, fields): return "red" if fields["state"] == "FAIL" else "white" def store_data(self, data): self.data = data["runs"] self.job_run = data def rows(self): # Action runs need a sort order that sorts by date # and that can handle situations where it is None, or # othere weird things, so we str() return sorted( self.data, key=lambda x: str(x[self.fields[self.sort_index]]), reverse=self.reversed, ) def display_node(source, _=None): if not source: return "" return "{}@{}".format(source["username"], source["hostname"]) def display_node_pool(source, _=None): if not source: return "" return "%s (%d node(s))" % (source["name"], len(source["nodes"])) def display_scheduler(source, _=None): if not source: return "" return "{} {}{}".format(source["type"], source["value"], source["jitter"]) def display_state_delayed(_, obj): state = obj["state"] in_delay = obj["in_delay"] if in_delay: return f"{state} (retry delayed for {int(in_delay)}s)" else: return state field_display_mapping = { "node": display_node, "node_pool": display_node_pool, "scheduler": display_scheduler, "state_delayed": display_state_delayed, "exit_status": lambda v, _: exitcode.EXIT_REASONS.get(v, v), } def view_with_less(content, color=True): """Send `content` through less.""" import subprocess cmd = ["less"] if color: cmd.append("-r") less_proc = subprocess.Popen(cmd, stdin=subprocess.PIPE) less_proc.stdin.write( maybe_encode(content) ) # TODO: TRON-2293 maybe_encode is a relic of Python2->Python3 migration. Remove it. less_proc.stdin.close() less_proc.wait() ================================================ FILE: tron/commands/retry.py ================================================ import argparse import asyncio import datetime import functools import random from urllib.parse import urljoin import pytimeparse # type:ignore from tron.commands import client from tron.commands import display from tron.commands.backfill import BackfillRun DEFAULT_POLLING_INTERVAL_S = 10 def parse_deps_timeout(duration: str) -> int: if duration == "infinity": return RetryAction.WAIT_FOREVER elif duration.isnumeric(): seconds = int(duration) else: seconds = pytimeparse.parse(duration) if seconds is None: raise argparse.ArgumentTypeError( f"'{duration}' is not a valid duration. Must be either number of seconds or pytimeparse-parsable string." ) if seconds < 0: raise argparse.ArgumentTypeError(f"'{duration}' must not be negative") return seconds class RetryAction: NO_TIMEOUT = 0 WAIT_FOREVER = -1 RETRY_NOT_ISSUED = None RETRY_SUCCESS = True RETRY_FAIL = False def __init__( self, tron_client: client.Client, full_action_name: str, use_latest_command: bool = False, ): self.tron_client = tron_client self.retry_params = dict(command="retry", use_latest_command=int(use_latest_command)) self.full_action_name = full_action_name self.action_run_id = self._validate_action_name(full_action_name) self.job_run_id = client.get_object_type_from_identifier(self.tron_client.index(), self.job_run_name) self._required_action_indices = self._get_required_action_indices() self._elapsed = datetime.timedelta(seconds=0) self._triggers_done = False self._required_actions_done = False self._retry_request_result: bool | None = RetryAction.RETRY_NOT_ISSUED @property def job_run_name(self) -> str: return self.full_action_name.rsplit(".", 1)[0] @property def action_name(self) -> str: return self.full_action_name.rsplit(".", 1)[1] @property def status(self) -> str: if not self._triggers_done: return "Upstream triggers not all published" elif not self._required_actions_done: return "Required actions not all successfully completed" elif self._retry_request_result == RetryAction.RETRY_NOT_ISSUED: return "Retry request not issued, but dependencies done" elif self._retry_request_result == RetryAction.RETRY_SUCCESS: return "Retry request issued successfully" else: return "Failed to issue retry request" @property def succeeded(self) -> bool: return bool(self._retry_request_result) def _validate_action_name(self, full_action_name: str) -> client.TronObjectIdentifier: action_run_id: client.TronObjectIdentifier = client.get_object_type_from_identifier( self.tron_client.index(), full_action_name ) if action_run_id.type != client.TronObjectType.action_run: raise ValueError(f"'{full_action_name}' is a {action_run_id.type.lower()}, not an action") self.tron_client.action_runs(action_run_id.url, num_lines=0) # verify action exists return action_run_id def _get_required_action_indices(self) -> dict[str, int]: job_run = self.tron_client.job_runs(self.job_run_id.url) required_actions = set() action_indices = {} for i, action_run in enumerate(job_run["runs"]): if action_run["action_name"] == self.action_name: required_actions = set(action_run["requirements"]) action_indices[action_run["action_name"]] = i return {action_name: i for action_name, i in action_indices.items() if action_name in required_actions} def _log(self, msg: str) -> None: print(f"[{self._elapsed}] {self.full_action_name}: {msg}") async def can_retry(self) -> bool: if not self._triggers_done: triggers = await self.check_trigger_statuses() self._triggers_done = all(triggers.values()) if self._triggers_done: if len(triggers) > 0: self._log("All upstream triggers published") else: remaining_triggers = [trigger for trigger, is_done in triggers.items() if not is_done] self._log(f"Upstream triggers not yet published: {remaining_triggers}") if not self._required_actions_done: required_actions = await self.check_required_actions_statuses() self._required_actions_done = all(required_actions.values()) if self._required_actions_done: if len(required_actions) > 0: self._log("All required actions finished") else: remaining_required_actions = [action for action, is_done in required_actions.items() if not is_done] self._log(f"Required actions not yet succeeded: {remaining_required_actions}") return self._triggers_done and self._required_actions_done async def check_trigger_statuses(self) -> dict[str, bool]: action_run = await asyncio.get_event_loop().run_in_executor( None, functools.partial( self.tron_client.action_runs, self.action_run_id.url, num_lines=0, ), ) # from tron.api.adapter:ActionRunAdapter.get_triggered_by: # triggered_by is a single string with this format: # {trigger_1} (done), {trigger_2}, etc. # where trigger_1 has been published, and trigger_2 is still waiting trigger_states = {} for trigger_and_state in action_run["triggered_by"].split(", "): if trigger_and_state: trigger, *maybe_state = trigger_and_state.split(" ") # if len(parts) == 2, then parts is [{trigger}, "(done)"] # else, parts is [{trigger}] trigger_states[trigger] = len(maybe_state) == 1 return trigger_states async def check_required_actions_statuses(self) -> dict[str, bool]: action_runs = ( await asyncio.get_event_loop().run_in_executor( None, self.tron_client.job_runs, self.job_run_id.url, ) )["runs"] return { action_runs[i]["action_name"]: action_runs[i]["state"] in BackfillRun.SUCCESS_STATES for i in self._required_action_indices.values() } async def wait_and_retry( self, deps_timeout_s: int = 0, poll_interval_s: int = DEFAULT_POLLING_INTERVAL_S, jitter: bool = True, ) -> bool: if deps_timeout_s != RetryAction.NO_TIMEOUT and jitter: init_delay_s = random.randint(1, min(deps_timeout_s, poll_interval_s)) - 1 self._elapsed += datetime.timedelta(seconds=init_delay_s) await asyncio.sleep(init_delay_s) if await self.wait_for_deps(deps_timeout_s=deps_timeout_s, poll_interval_s=poll_interval_s): return await self.issue_retry() else: deps_timeout_td = datetime.timedelta(seconds=deps_timeout_s) msg = "Action will not be retried." if deps_timeout_s != RetryAction.NO_TIMEOUT: msg = f"Not all dependencies completed after waiting for {deps_timeout_td}. " + msg self._log(msg) return False async def wait_for_deps( self, deps_timeout_s: int = 0, poll_interval_s: int = DEFAULT_POLLING_INTERVAL_S, ) -> bool: """Wait for all upstream dependencies to finished up to a timeout. Once the timeout has expired, one final check is always conducted. Returns whether or not deps successfully finished. """ while deps_timeout_s == RetryAction.WAIT_FOREVER or self._elapsed.seconds < deps_timeout_s: if await self.can_retry(): return True wait_for = poll_interval_s if deps_timeout_s != RetryAction.WAIT_FOREVER: wait_for = min(wait_for, int(deps_timeout_s - self._elapsed.seconds)) await asyncio.sleep(wait_for) self._elapsed += datetime.timedelta(seconds=wait_for) return await self.can_retry() async def issue_retry(self) -> bool: self._log("Issuing retry request") response = await asyncio.get_event_loop().run_in_executor( None, functools.partial( client.request, urljoin(self.tron_client.url_base, self.action_run_id.url), data=self.retry_params, user_attribution=True, ), ) if response.error: self._log(f"Error: couldn't issue retry request: {response.content}") self._retry_request_result = RetryAction.RETRY_FAIL else: self._log(f"Got result: {response.content.get('result')}") self._log(f"Check the status of the retry run using: `tronview {self.full_action_name}`") self._retry_request_result = RetryAction.RETRY_SUCCESS return self._retry_request_result def retry_actions( tron_server: str, full_action_names: list[str], use_latest_command: bool = False, deps_timeout_s: int = RetryAction.NO_TIMEOUT, ) -> list[RetryAction]: tron_client = client.Client(tron_server, user_attribution=True) r_actions = [RetryAction(tron_client, name, use_latest_command=use_latest_command) for name in full_action_names] loop = asyncio.get_event_loop() try: # first action starts checking immediately, rest have a jitter loop.run_until_complete( asyncio.gather( r_actions[0].wait_and_retry(deps_timeout_s=deps_timeout_s, jitter=False), *[ra.wait_and_retry(deps_timeout_s=deps_timeout_s) for ra in r_actions[1:]], ) ) finally: loop.close() return r_actions class DisplayRetries(display.TableDisplay): columns = ["Action Name", "Final Status"] fields = ["full_action_name", "status"] widths = [60, 60] title = "Retries" resize_fields = {"full_action_name", "status"} header_color = "hgray" def print_retries_table(retries: list[RetryAction]) -> None: """Prints retry runs in a table""" with display.Color.enable(): table = DisplayRetries().format([dict(full_action_name=r.full_action_name, status=r.status) for r in retries]) print(table) ================================================ FILE: tron/config/__init__.py ================================================ class ConfigError(Exception): """Generic exception class for errors with config validation""" pass ================================================ FILE: tron/config/config_parse.py ================================================ """ Parse a dictionary structure and return an immutable structure that contain a validated configuration. WARNING: it is *NOT* safe to delete classes that are being validated (or their attributes) if there are any references to them in DynamoDB until TRON-2200 is complete! (See DAR-2328) NOTE: this means that reverting a change that adds a new attribute is not safe :) """ import datetime import getpass import itertools import logging import os from copy import deepcopy from typing import Any from urllib.parse import urlparse import pytz from task_processing.plugins.mesos.constraints import OPERATORS from tron import command_context from tron.config import config_utils from tron.config import ConfigError from tron.config import schema from tron.config.config_utils import build_dict_name_validator from tron.config.config_utils import build_dict_value_validator from tron.config.config_utils import build_list_of_type_validator from tron.config.config_utils import ConfigContext from tron.config.config_utils import PartialConfigContext from tron.config.config_utils import StringFormatter from tron.config.config_utils import valid_bool from tron.config.config_utils import valid_dict from tron.config.config_utils import valid_exit_code from tron.config.config_utils import valid_float from tron.config.config_utils import valid_identifier from tron.config.config_utils import valid_int from tron.config.config_utils import valid_list from tron.config.config_utils import valid_name_identifier from tron.config.config_utils import valid_string from tron.config.config_utils import Validator from tron.config.schedule_parse import valid_schedule from tron.config.schema import CLEANUP_ACTION_NAME from tron.config.schema import ConfigAction from tron.config.schema import ConfigCleanupAction from tron.config.schema import ConfigConstraint from tron.config.schema import ConfigFieldSelectorSource from tron.config.schema import ConfigJob from tron.config.schema import ConfigKubernetes from tron.config.schema import ConfigMesos from tron.config.schema import ConfigNodeAffinity from tron.config.schema import ConfigParameter from tron.config.schema import ConfigProjectedSAVolume from tron.config.schema import ConfigSecretSource from tron.config.schema import ConfigSecretVolume from tron.config.schema import ConfigSecretVolumeItem from tron.config.schema import ConfigSSHOptions from tron.config.schema import ConfigState from tron.config.schema import ConfigTopologySpreadConstraints from tron.config.schema import ConfigVolume from tron.config.schema import MASTER_NAMESPACE from tron.config.schema import NamedTronConfig from tron.config.schema import TronConfig log = logging.getLogger(__name__) def build_format_string_validator(context_object): """Validate that a string does not contain any unexpected formatting keys. valid_keys - a sequence of strings """ def validator(value, config_context): if config_context.partial: return valid_string(value, config_context) context = command_context.CommandContext( context_object, config_context.command_context, ) try: StringFormatter(context).format(value) return value except (KeyError, ValueError) as e: error_msg = "Unknown context variable %s at %s: %s" raise ConfigError(error_msg % (e, config_context.path, value)) except TypeError as e: error_msg = "Wrong command format %s: %s at %s" raise ConfigError(error_msg % (value, e, config_context.path)) return validator def valid_output_stream_dir(output_dir, config_context): """Returns a valid string for the output directory, or raises ConfigError if the output_dir is not valid. """ if not output_dir: return if config_context.partial: return output_dir valid_string(output_dir, config_context) if not os.path.isdir(output_dir): msg = "output_stream_dir '%s' is not a directory" raise ConfigError(msg % output_dir) if not os.access(output_dir, os.W_OK): raise ConfigError( "output_stream_dir '%s' is not writable" % output_dir, ) return output_dir def valid_identity_file(file_path, config_context): valid_string(file_path, config_context) if config_context.partial: return file_path file_path = os.path.expanduser(file_path) if not os.path.exists(file_path): raise ConfigError("Private key file %s doesn't exist" % file_path) public_key_path = file_path + ".pub" if not os.path.exists(public_key_path): raise ConfigError("Public key file %s doesn't exist" % public_key_path) return file_path def valid_known_hosts_file(file_path, config_context): valid_string(file_path, config_context) if config_context.partial: return file_path file_path = os.path.expanduser(file_path) if not os.path.exists(file_path): raise ConfigError("Known hosts file %s doesn't exist" % file_path) return file_path def valid_command_context(context, config_context): # context can be any dict. return valid_dict(context or {}, config_context) def valid_time_zone(tz, config_context): if tz is None: return None valid_string(tz, config_context) try: return pytz.timezone(tz) except pytz.exceptions.UnknownTimeZoneError: raise ConfigError("%s is not a valid time zone" % tz) def valid_node_name(value, config_context): valid_identifier(value, config_context) if not config_context.partial and value not in config_context.nodes: msg = "Unknown node name %s at %s" raise ConfigError(msg % (value, config_context.path)) return value def valid_master_address(value, config_context): """Validates and normalizes Mesos master address. Must be HTTP or not include a scheme, and only include a host, without any path components. """ valid_string(value, config_context) # Parse with HTTP as default, only HTTP allowed. scheme, netloc, path, params, query, fragment = urlparse(value, "http") if scheme != "http": msg = f"Only HTTP supported for Mesos master address, got {value}" raise ConfigError(msg) if params or query or fragment: msg = f"Mesos master address may not contain path components, got {value}" raise ConfigError(msg) # Only one of netloc or path allowed, and no / except trailing ones. # netloc is empty if there's no scheme, then we try the path. path = path.rstrip("/") if (netloc and path) or "/" in path: msg = f"Mesos master address may not contain path components, got {value}" raise ConfigError(msg) if not netloc: netloc = path if not netloc: msg = f"Mesos master address is missing host, got {value}" raise ConfigError(msg) return f"{scheme}://{netloc}" def valid_k8s_master_address(value: str, config_context: ConfigContext) -> str: """Validates and normalizes Kubernetes master address. Must be HTTP or not include a scheme, and only include a host, without any path components. """ valid_string(value, config_context) # Parse with HTTP as default, only HTTPS allowed. scheme, netloc, path, params, query, fragment = urlparse(url=value, scheme="https") if scheme != "http": msg = f"Only HTTPS supported for Kubernetes master address, got {value}" raise ConfigError(msg) if params or query or fragment: msg = f"Kubernetes master address may not contain path components, got {value}" raise ConfigError(msg) # Only one of netloc or path allowed, and no / except trailing ones. path = path.rstrip("/") if (netloc and path) or "/" in path: msg = f"Kubernetes master address may not contain path components, got {value}" raise ConfigError(msg) # netloc is empty if there's no scheme, so we fallback to path. if not netloc and path: netloc = path if not netloc: msg = f"Kubernetes master address is missing host, got {value}" raise ConfigError(msg) return f"{scheme}://{netloc}" class ValidateConstraint(Validator): config_class = ConfigConstraint validators = { "attribute": valid_string, "operator": config_utils.build_enum_validator(OPERATORS.keys()), "value": valid_string, } valid_constraint = ValidateConstraint() class ValidateDockerParameter(Validator): config_class = ConfigParameter validators = { "key": valid_string, "value": valid_string, } valid_docker_parameter = ValidateDockerParameter() class ValidateVolume(Validator): config_class = ConfigVolume validators = { "container_path": valid_string, "host_path": valid_string, "mode": config_utils.build_real_enum_validator(schema.VolumeModes), } valid_volume = ValidateVolume() class ValidateSecretSource(Validator): config_class = ConfigSecretSource validators = { "secret_name": valid_string, # name of Kubernetes Secret "key": valid_string, # key name in Secret data } valid_secret_source = ValidateSecretSource() def valid_permission_mode(value: str | int, config_context: ConfigContext) -> str: try: decimal_value = int( str(value), base=8 ) # take in permission mode as string or int representation of an octal number. Goes from 0 to 4095 in decimal. except ValueError: error_msg = "Could not parse {} as octal permission mode at {}" raise ConfigError(error_msg.format(value, config_context.path)) if decimal_value > 4095 or decimal_value < 0: error_msg = "Octal permission mode {} out of bound at {}" raise ConfigError(error_msg.format(value, config_context.path)) return str(value) class ValidateSecretVolumeItem(Validator): config_class = ConfigSecretVolumeItem validators = { "key": valid_string, # name of current secret "path": valid_string, # New secret filename "mode": valid_permission_mode, # Octal permission mode } valid_secret_volume_item = ValidateSecretVolumeItem() class ValidateSecretVolume(Validator): config_class = ConfigSecretVolume optional = True defaults = { "default_mode": "0644", "items": None, } validators = { "container_path": valid_string, "secret_volume_name": valid_string, "secret_name": valid_string, "default_mode": valid_permission_mode, "items": build_list_of_type_validator(valid_secret_volume_item, allow_empty=True), } def post_validation(self, valid_input, config_context): """Propagate default mode and enforce the secret-key match.""" # Ensure 'items' is an iterable list, even if defaulted to None by set_defaults. # The 'or []' handles the case where valid_input.get('items') returns None. items = valid_input.get("items") or [] # Our secrets will really only ever have one key, so weirdly we only care about a single # item of this array AND it must have the same name as the secret (which is the single key). if len(items) > 1: raise ConfigError( "There is more than one item in the items array. This is unsupported as we don't support multi-key secrets." ) processed_items = [] modified = False default_mode = valid_input.get("default_mode", self.defaults["default_mode"]) secret_name = valid_input.get("secret_name") for item in items: if item.key != secret_name: raise ConfigError(f"Item key '{item.key}' does not match the volume's secret name '{secret_name}'") final_item = item if item.mode is None: # Apply volume's default_mode to items without an explicit mode. final_item = item._replace(mode=default_mode) modified = True processed_items.append(final_item) if modified: # Update valid_input with the (potentially) modified items tuple. # This ensures the final object reflects applied defaults. valid_input["items"] = tuple(processed_items) valid_secret_volume = ValidateSecretVolume() class ValidateProjectedSAVolume(Validator): config_class = ConfigProjectedSAVolume optional = True defaults = { "expiration_seconds": 1800, } validators = { "container_path": valid_string, "audience": valid_string, "expiration_seconds": valid_int, } valid_projected_sa_volume = ValidateProjectedSAVolume() class ValidateFieldSelectorSource(Validator): config_class = ConfigFieldSelectorSource validators = { "field_path": valid_string, # k8s field path - e.g., `status.podIP` } valid_field_selector_source = ValidateFieldSelectorSource() def _valid_node_affinity_operator(value: str, config_context: ConfigContext) -> str: valid_operators = {"In", "NotIn", "Exists", "NotExists", "Gt", "Lt"} if value not in valid_operators: raise ConfigError(f"Got {value} as a node affinity operator, expected one of {valid_operators}") return value class ValidateNodeAffinity(Validator): config_class = ConfigNodeAffinity validators = { "key": valid_string, "operator": _valid_node_affinity_operator, "value": build_list_of_type_validator(valid_string, allow_empty=True), } valid_node_affinity = ValidateNodeAffinity() def _valid_when_unsatisfiable(value: str, config_context: ConfigContext) -> str: valid_values = {"DoNotSchedule", "ScheduleAnyway"} if value not in valid_values: raise ConfigError(f"Got {value} as a when_unsatisfiable value, expected one of {valid_values}") return value def _valid_topology_spread_label_selector(value: dict[str, str], config_context: ConfigContext) -> dict[str, str]: if not value: raise ConfigError("TopologySpreadConstraints must have a label_selector") # XXX: we probably also want to enforce k8s limits for label lengths and whatnot if not all(isinstance(k, str) for k in value.keys()): raise ConfigError("TopologySpreadConstraints label_selector keys must be strings") if not all(isinstance(s, str) for s in value.values()): raise ConfigError("TopologySpreadConstraints label_selector values must be strings") return value class ValidateTopologySpreadConstraints(Validator): config_class = ConfigTopologySpreadConstraints validators = { "max_skew": valid_int, "when_unsatisfiable": _valid_when_unsatisfiable, "topology_key": valid_string, "label_selector": _valid_topology_spread_label_selector, } valid_topology_spread_constraints = ValidateTopologySpreadConstraints() class ValidateSSHOptions(Validator): """Validate SSH options.""" config_class = ConfigSSHOptions optional = True defaults = { "agent": False, "identities": (), "known_hosts_file": None, "connect_timeout": 30, "idle_connection_timeout": 3600, "jitter_min_load": 4, "jitter_max_delay": 20, "jitter_load_factor": 1, } validators = { "agent": valid_bool, # TODO: move this config and validations outside master namespace # 'identities': build_list_of_type_validator( # valid_identity_file, allow_empty=True), "identities": build_list_of_type_validator( valid_string, allow_empty=True, ), # 'known_hosts_file': valid_known_hosts_file, "known_hosts_file": valid_string, "connect_timeout": config_utils.valid_int, "idle_connection_timeout": config_utils.valid_int, "jitter_min_load": config_utils.valid_int, "jitter_max_delay": config_utils.valid_int, "jitter_load_factor": config_utils.valid_int, } def post_validation(self, valid_input, config_context): if config_context.partial: return if valid_input["agent"] and "SSH_AUTH_SOCK" not in os.environ: raise ConfigError("No SSH Agent available ($SSH_AUTH_SOCK)") valid_ssh_options = ValidateSSHOptions() class ValidateNode(Validator): config_class = schema.ConfigNode validators = { "name": config_utils.valid_identifier, "username": config_utils.valid_string, "hostname": config_utils.valid_string, "port": config_utils.valid_int, } defaults = { "port": 22, "username": getpass.getuser(), } def do_shortcut(self, node): """Nodes can be specified with just a hostname string.""" if isinstance(node, str): return schema.ConfigNode(hostname=node, name=node, **self.defaults) def set_defaults(self, output_dict, config_context): super().set_defaults(output_dict, config_context) output_dict.setdefault("name", output_dict["hostname"]) valid_node = ValidateNode() class ValidateNodePool(Validator): config_class = schema.ConfigNodePool validators = { "name": valid_identifier, "nodes": build_list_of_type_validator(valid_identifier), } def cast(self, node_pool, _context): if isinstance(node_pool, list): node_pool = dict(nodes=node_pool) return node_pool def set_defaults(self, node_pool, _): node_pool.setdefault("name", "_".join(node_pool["nodes"])) valid_node_pool = ValidateNodePool() def valid_action_name(value, config_context): valid_identifier(value, config_context) if value == CLEANUP_ACTION_NAME: error_msg = "Invalid action name %s at %s" raise ConfigError(error_msg % (value, config_context.path)) return value action_context = command_context.build_filled_context( command_context.JobContext, command_context.JobRunContext, command_context.ActionRunContext, ) def valid_mesos_action(action, config_context): required_keys = {"cpus", "mem", "docker_image"} if action.get("executor") == schema.ExecutorTypes.mesos.value: missing_keys = required_keys - set(action.keys()) if missing_keys: raise ConfigError( "Mesos executor for action {id} is missing these required keys: {keys}".format( id=action["name"], keys=missing_keys, ), ) def valid_kubernetes_action(action, config_context): required_keys = {"cpus", "mem", "docker_image"} if action.get("executor") == schema.ExecutorTypes.kubernetes.value: missing_keys = required_keys - set(action.keys()) if missing_keys: raise ConfigError( "Kubernetes executor for action {id} is missing these required keys: {keys}".format( id=action["name"], keys=missing_keys, ), ) def valid_trigger_downstreams(trigger_downstreams, config_context): if isinstance(trigger_downstreams, (type(None), bool, dict)): return trigger_downstreams raise ConfigError("must be None, bool or dict") class ValidateAction(Validator): """Validate an action.""" config_class = ConfigAction defaults = { "node": None, "requires": (), "retries": None, "retries_delay": None, "expected_runtime": datetime.timedelta(hours=24), "executor": schema.ExecutorTypes.ssh.value, "cpus": None, "mem": None, "disk": None, "cap_add": None, "cap_drop": None, "constraints": None, "docker_image": None, "docker_parameters": None, "env": None, "secret_env": None, "secret_volumes": None, "projected_sa_volumes": None, "field_selector_env": None, "extra_volumes": None, "trigger_downstreams": None, "triggered_by": None, "on_upstream_rerun": None, "trigger_timeout": None, "node_selectors": None, "node_affinities": None, "topology_spread_constraints": None, "labels": None, "annotations": None, "service_account_name": None, "ports": None, } requires = build_list_of_type_validator( valid_action_name, allow_empty=True, ) validators = { "name": valid_action_name, "command": build_format_string_validator(action_context), "node": valid_node_name, "requires": requires, "retries": valid_int, "retries_delay": config_utils.valid_time_delta, "expected_runtime": config_utils.valid_time_delta, "executor": config_utils.build_real_enum_validator(schema.ExecutorTypes), "cpus": valid_float, "mem": valid_float, "disk": valid_float, "cap_add": valid_list, "cap_drop": valid_list, "constraints": build_list_of_type_validator(valid_constraint, allow_empty=True), "docker_image": valid_string, "docker_parameters": build_list_of_type_validator( valid_docker_parameter, allow_empty=True, ), "env": valid_dict, "secret_env": build_dict_value_validator(valid_secret_source), "secret_volumes": build_list_of_type_validator(valid_secret_volume, allow_empty=True), "projected_sa_volumes": build_list_of_type_validator(valid_projected_sa_volume, allow_empty=True), "field_selector_env": build_dict_value_validator(valid_field_selector_source), "extra_volumes": build_list_of_type_validator(valid_volume, allow_empty=True), "trigger_downstreams": valid_trigger_downstreams, "triggered_by": build_list_of_type_validator(valid_string, allow_empty=True), "on_upstream_rerun": config_utils.build_real_enum_validator(schema.ActionOnRerun), "trigger_timeout": config_utils.valid_time_delta, "node_selectors:": valid_dict, "node_affinities": build_list_of_type_validator(valid_node_affinity, allow_empty=True), "topology_spread_constraints": build_list_of_type_validator( valid_topology_spread_constraints, allow_empty=True ), "labels:": valid_dict, "annotations": valid_dict, "service_account_name": valid_string, "ports": build_list_of_type_validator(valid_int, allow_empty=True), } def post_validation(self, action, config_context): valid_mesos_action(action, config_context) valid_kubernetes_action(action, config_context) valid_action = ValidateAction() def valid_cleanup_action_name(value, config_context): if value != CLEANUP_ACTION_NAME: msg = "Cleanup actions cannot have custom names %s.%s" raise ConfigError(msg % (config_context.path, value)) return CLEANUP_ACTION_NAME class ValidateCleanupAction(Validator): config_class = ConfigCleanupAction defaults = { "node": None, "name": CLEANUP_ACTION_NAME, "retries": None, "retries_delay": None, "expected_runtime": datetime.timedelta(hours=24), "executor": schema.ExecutorTypes.ssh.value, "cpus": None, "mem": None, "disk": None, "cap_add": None, "cap_drop": None, "constraints": None, "docker_image": None, "docker_parameters": None, "env": None, "secret_env": None, "secret_volumes": None, "projected_sa_volumes": None, "field_selector_env": None, "extra_volumes": None, "trigger_downstreams": None, "triggered_by": None, "on_upstream_rerun": None, "trigger_timeout": None, "node_selectors": None, "node_affinities": None, "topology_spread_constraints": None, "labels": None, "annotations": None, "service_account_name": None, "ports": None, } validators = { "name": valid_cleanup_action_name, "command": build_format_string_validator(action_context), "node": valid_node_name, "retries": valid_int, "retries_delay": config_utils.valid_time_delta, "expected_runtime": config_utils.valid_time_delta, "executor": config_utils.build_real_enum_validator(schema.ExecutorTypes), "cpus": valid_float, "mem": valid_float, "disk": valid_float, "cap_add": valid_list, "cap_drop": valid_list, "constraints": build_list_of_type_validator(valid_constraint, allow_empty=True), "docker_image": valid_string, "docker_parameters": build_list_of_type_validator( valid_docker_parameter, allow_empty=True, ), "env": valid_dict, "secret_env": build_dict_value_validator(valid_secret_source), "secret_volumes": build_list_of_type_validator(valid_secret_volume, allow_empty=True), "projected_sa_volumes": build_list_of_type_validator(valid_projected_sa_volume, allow_empty=True), "field_selector_env": build_dict_value_validator(valid_field_selector_source), "extra_volumes": build_list_of_type_validator(valid_volume, allow_empty=True), "trigger_downstreams": valid_trigger_downstreams, "triggered_by": build_list_of_type_validator(valid_string, allow_empty=True), "on_upstream_rerun": config_utils.build_real_enum_validator(schema.ActionOnRerun), "trigger_timeout": config_utils.valid_time_delta, "node_selectors:": valid_dict, "node_affinities": build_list_of_type_validator(valid_node_affinity, allow_empty=True), "topology_spread_constraints": build_list_of_type_validator( valid_topology_spread_constraints, allow_empty=True ), "labels": valid_dict, "annotations": valid_dict, "service_account_name": valid_string, "ports": build_list_of_type_validator(valid_int, allow_empty=True), } def post_validation(self, action, config_context): valid_mesos_action(action, config_context) valid_kubernetes_action(action, config_context) valid_cleanup_action = ValidateCleanupAction() class ValidateJob(Validator): """Validate jobs.""" config_class = ConfigJob defaults: dict[str, Any] = { "run_limit": 50, "all_nodes": False, "cleanup_action": None, "enabled": True, "queueing": True, "allow_overlap": False, "max_runtime": None, "monitoring": {}, "time_zone": None, "expected_runtime": datetime.timedelta(hours=24), "use_k8s": False, } validators = { "name": valid_name_identifier, "schedule": valid_schedule, "run_limit": valid_int, "all_nodes": valid_bool, "actions": build_dict_name_validator(valid_action), "cleanup_action": valid_cleanup_action, "node": valid_node_name, "queueing": valid_bool, "enabled": valid_bool, "allow_overlap": valid_bool, "max_runtime": config_utils.valid_time_delta, "monitoring": valid_dict, "time_zone": valid_time_zone, "expected_runtime": config_utils.valid_time_delta, "use_k8s": valid_bool, } def cast(self, in_dict, config_context): in_dict["namespace"] = config_context.namespace return in_dict # TODO: extract common code to a util function def _validate_dependencies( self, job: dict[str, Any], # TODO: create TypedDict for this # TODO: setup UniqueNameDict for use with mypy so that the following line # is not a lie actions: dict[str, ConfigAction], base_action: ConfigAction, current_action: ConfigAction | None = None, stack: list[str] | None = None, already_validated: set[tuple[str, str]] | None = None, ) -> None: """Check for circular or misspelled dependencies.""" # for large graphs, we can end up validating the same jobs/actions repeatedly # this is unnecessary and we can skip a ton of work simply by caching what we've # already validated already_validated = already_validated or set() current_action = current_action or base_action validated = (job["name"], current_action.name) if validated in already_validated: return None else: already_validated.add(validated) stack = stack or [] stack.append(current_action.name) for dep in current_action.requires: if dep == base_action.name and len(stack) > 0: msg = "Circular dependency in job.%s: %s" raise ConfigError(msg % (job["name"], " -> ".join(stack))) if dep not in actions: raise ConfigError( 'Action jobs.%s.%s has a dependency "%s"' " that is not in the same job!" % (job["name"], current_action.name, dep), ) self._validate_dependencies(job, actions, base_action, actions[dep], stack, already_validated) stack.pop() def post_validation(self, job, config_context): """Validate actions for the job.""" for _, action in job["actions"].items(): self._validate_dependencies(job, job["actions"], action) valid_job = ValidateJob() class ValidateActionRunner(Validator): config_class = schema.ConfigActionRunner optional = True defaults = { "runner_type": None, "remote_exec_path": "", "remote_status_path": "/tmp", } validators = { "runner_type": config_utils.build_real_enum_validator(schema.ActionRunnerTypes), "remote_status_path": valid_string, "remote_exec_path": valid_string, } class ValidateStatePersistence(Validator): config_class = schema.ConfigState defaults = { "buffer_size": 1, "dynamodb_region": None, "table_name": None, "max_transact_write_items": 8, } validators = { "name": valid_string, "store_type": config_utils.build_real_enum_validator(schema.StatePersistenceTypes), "buffer_size": valid_int, "dynamodb_region": valid_string, "table_name": valid_string, "max_transact_write_items": valid_int, } def post_validation(self, config, config_context): buffer_size = config.get("buffer_size") if buffer_size and buffer_size < 1: path = config_context.path raise ConfigError("%s buffer_size must be >= 1." % path) store_type = config.get("store_type") if store_type == schema.StatePersistenceTypes.dynamodb.value: if not config.get("table_name"): raise ConfigError(f"{config_context.path} table_name is required when store_type is 'dynamodb'") if not config.get("dynamodb_region"): raise ConfigError(f"{config_context.path} dynamodb_region is required when store_type is 'dynamodb'") max_transact = config.get("max_transact_write_items") # Upper bound is based on boto3 transact_write_items limit if not 1 <= max_transact <= 100: raise ConfigError( f"{config_context.path} max_transact_write_items must be between 1 and 100, got {max_transact}" ) valid_state_persistence = ValidateStatePersistence() class ValidateMesos(Validator): config_class = ConfigMesos option = True defaults = { "master_address": None, "master_port": 5050, "secret_file": None, "role": "*", "principal": "tron", "enabled": False, "default_volumes": (), "dockercfg_location": None, "offer_timeout": 300, } validators = { "master_address": valid_master_address, "master_port": valid_int, "secret": valid_string, "role": valid_string, "enabled": valid_bool, "default_volumes": build_list_of_type_validator(valid_volume, allow_empty=True), "dockercfg_location": valid_string, "offer_timeout": valid_int, } valid_mesos_options = ValidateMesos() class ValidateKubernetes(Validator): config_class = ConfigKubernetes optional = True defaults = { "kubeconfig_path": None, "enabled": False, "non_retryable_exit_codes": (), "default_volumes": (), } validators = { "kubeconfig_path": valid_string, "enabled": valid_bool, "non_retryable_exit_codes": build_list_of_type_validator(valid_exit_code, allow_empty=True), "default_volumes": build_list_of_type_validator(valid_volume, allow_empty=True), "watcher_kubeconfig_paths": build_list_of_type_validator(valid_string, allow_empty=True), } valid_kubernetes_options = ValidateKubernetes() def validate_jobs(config, config_context): """Validate jobs""" valid_jobs = build_dict_name_validator(valid_job, allow_empty=True) validation = [("jobs", valid_jobs)] for config_name, valid in validation: child_context = config_context.build_child_context(config_name) config[config_name] = valid(config.get(config_name, []), child_context) fmt_string = "Job names must be unique %s" config_utils.unique_names(fmt_string, config["jobs"]) DEFAULT_STATE_PERSISTENCE = ConfigState( name="tron_state", store_type="shelve", buffer_size=1, ) DEFAULT_NODE = ValidateNode().do_shortcut(node="localhost") class ValidateConfig(Validator): """Given a parsed config file (should be only basic literals and containers), return an immutable, fully populated series of namedtuples and dicts with all defaults filled in, all valid values, and no unused values. Throws a ConfigError if any part of the input dict is invalid. """ config_class = TronConfig defaults = { "action_runner": {}, "output_stream_dir": None, "command_context": {}, "ssh_options": ConfigSSHOptions(**ValidateSSHOptions.defaults), "time_zone": None, "state_persistence": DEFAULT_STATE_PERSISTENCE, "nodes": { "localhost": DEFAULT_NODE, }, "node_pools": {}, "jobs": (), "mesos_options": ConfigMesos(**ValidateMesos.defaults), "k8s_options": ConfigKubernetes(**ValidateKubernetes.defaults), "eventbus_enabled": None, "read_json": False, } node_pools = build_dict_name_validator(valid_node_pool, allow_empty=True) nodes = build_dict_name_validator(valid_node, allow_empty=True) validators = { "action_runner": ValidateActionRunner(), "output_stream_dir": valid_output_stream_dir, "command_context": valid_command_context, "ssh_options": valid_ssh_options, "time_zone": valid_time_zone, "state_persistence": valid_state_persistence, "nodes": nodes, "node_pools": node_pools, "mesos_options": valid_mesos_options, "k8s_options": valid_kubernetes_options, "eventbus_enabled": valid_bool, "read_json": valid_bool, } optional = False def validate_node_pool_nodes(self, config): """Validate that each node in a node_pool is in fact a node, and not another pool. """ all_node_names = set(config["nodes"]) for node_pool in config["node_pools"].values(): invalid_names = set(node_pool.nodes) - all_node_names if invalid_names: msg = "NodePool %s contains other NodePools: " % node_pool.name raise ConfigError(msg + ",".join(invalid_names)) def post_validation(self, config, _): """Validate a non-named config.""" node_names = config_utils.unique_names( "Node and NodePool names must be unique %s", config["nodes"], config.get("node_pools", []), ) if config.get("node_pools"): self.validate_node_pool_nodes(config) config_context = ConfigContext( "config", node_names, config.get("command_context"), MASTER_NAMESPACE, ) validate_jobs(config, config_context) class ValidateNamedConfig(Validator): """A shorter validator for named configurations, which allow for jobs to be defined as configuration fragments that are, in turn, reconciled by Tron. """ config_class = NamedTronConfig type_name = "NamedConfigFragment" defaults = { "jobs": (), } optional = False def post_validation(self, config, config_context): validate_jobs(config, config_context) valid_config = ValidateConfig() valid_named_config = ValidateNamedConfig() def validate_fragment(name, fragment, master_config=None): """Validate a fragment with a partial context.""" config_context = PartialConfigContext(name, name) if name == MASTER_NAMESPACE: return valid_config(fragment, config_context=config_context) if master_config is None: return valid_named_config(fragment, config_context=config_context) config_mapping = {MASTER_NAMESPACE: master_config, name: fragment} for config_name, config in validate_config_mapping(config_mapping): if config_name == name: return config def get_nodes_from_master_namespace(master): return set(itertools.chain(master.nodes, master.node_pools)) def validate_config_mapping(config_mapping): if MASTER_NAMESPACE not in config_mapping: msg = "A config mapping requires a %s namespace" raise ConfigError(msg % MASTER_NAMESPACE) # we mutate this mapping - so let's make sure that we're making a copy # in case the passed-in mapping is used elsewhere config_mapping_to_validate = deepcopy(config_mapping) master = valid_config(config_mapping_to_validate.pop(MASTER_NAMESPACE)) nodes = get_nodes_from_master_namespace(master) yield MASTER_NAMESPACE, master for name, content in config_mapping_to_validate.items(): context = ConfigContext( name, nodes, master.command_context, name, ) yield name, valid_named_config(content, config_context=context) class ConfigContainer: """A container around configuration fragments (and master).""" def __init__(self, config_mapping): self.configs = config_mapping def items(self): return self.configs.items() @classmethod def create(cls, config_mapping): return cls(dict(validate_config_mapping(config_mapping))) # TODO: DRY with get_jobs() def get_job_names(self): job_names = [] for config in self.configs.values(): job_names.extend(config.jobs) return job_names def get_jobs(self): return dict( itertools.chain.from_iterable(config.jobs.items() for _, config in self.configs.items()), ) def get_master(self): return self.configs[MASTER_NAMESPACE] def get_node_names(self): return get_nodes_from_master_namespace(self.get_master()) def __getitem__(self, name): return self.configs[name] def __contains__(self, name): return name in self.configs ================================================ FILE: tron/config/config_utils.py ================================================ """Utilities used for configuration parsing and validation.""" import datetime import functools import itertools import re from string import Formatter from tron.config import ConfigError from tron.config.schema import MASTER_NAMESPACE MAX_IDENTIFIER_LENGTH = 255 IDENTIFIER_RE = re.compile(r"^[A-Za-z_][\w\-]{0,254}$") class StringFormatter(Formatter): def __init__(self, context=None): Formatter.__init__(self) self.context = context def get_value(self, key, args, kwds): if isinstance(key, str): try: return kwds[key] except KeyError: return self.context[key] else: return Formatter.get_value(key, args, kwds) class UniqueNameDict(dict): """A dict like object that throws a ConfigError if a key exists and __setitem__ is called to change the value of that key. fmt_string - format string used to create an error message, expects a single format argument of 'key' """ def __init__(self, fmt_string): super().__init__() self.fmt_string = fmt_string def __setitem__(self, key, value): if key in self: raise ConfigError(self.fmt_string % key) super().__setitem__(key, value) def unique_names(fmt_string, *seqs): """Validate that each object in all sequences has a unique name.""" name_dict = UniqueNameDict(fmt_string) for item in itertools.chain.from_iterable(seqs): name_dict[item] = True return name_dict def build_type_validator(validator, error_fmt): """Create a validator function using `validator` to validate the value. validator - a function which takes a single argument `value` error_fmt - a string which accepts two format variables (path, value) Returns a function func(value, config_context) where value - the value to validate config_context - a ConfigContext object Returns True if the value is valid """ def f(value, config_context): if not validator(value): raise ConfigError(error_fmt % (config_context.path, value)) return value return f def valid_number(type_func, value, config_context, allow_negative=False): path = config_context.path try: value = type_func(value) except TypeError: name = type_func.__name__ raise ConfigError(f"Value at {path} is not an {name}: {value}") if value < 0 and not allow_negative: raise ConfigError("%s must be a positive int." % path) return value valid_exit_code = functools.partial(valid_number, int, allow_negative=True) valid_int = functools.partial(valid_number, int) valid_float = functools.partial(valid_number, float) valid_identifier = build_type_validator( lambda s: isinstance(s, str) and IDENTIFIER_RE.match(s), "Identifier at %s is not a valid identifier: %s", ) valid_list = build_type_validator( lambda s: isinstance(s, list), "Value at %s is not a list: %s", ) valid_string = build_type_validator( lambda s: isinstance(s, str), "Value at %s is not a string: %s", ) valid_dict = build_type_validator( lambda s: isinstance(s, dict), "Value at %s is not a dictionary: %s", ) valid_bool = build_type_validator( lambda s: isinstance(s, bool), "Value at %s is not a boolean: %s", ) def build_enum_validator(enum): enum = set(enum) msg = "Value at %%s is not in %s: %%s." % str(enum) return build_type_validator(enum.__contains__, msg) def build_real_enum_validator(enum): def enum_validator(value, config_context): try: return enum(value).value except Exception: raise ConfigError( f"Value at {config_context.path} is not in {enum!r}: {value!r}", ) return enum_validator def valid_time(value, config_context): valid_string(value, config_context) for format in ["%H:%M", "%H:%M:%S"]: try: return datetime.datetime.strptime(value, format) except ValueError: pass msg = "Value at %s is not a valid time" raise ConfigError(msg % config_context.path) # Translations from possible configuration units to the argument to # datetime.timedelta TIME_INTERVAL_MAPPING = { "days": ["d", "day", "days"], "hours": ["h", "hr", "hrs", "hour", "hours"], "minutes": ["m", "min", "mins", "minute", "minutes"], "seconds": ["s", "sec", "secs", "second", "seconds"], } TIME_INTERVAL_UNITS = {short: long for (long, short_list) in TIME_INTERVAL_MAPPING.items() for short in short_list} TIME_INTERVAL_RE = re.compile(r"^\s*(?P\d+)\s*(?P[a-zA-Z]+)\s*$") def valid_time_delta(value, config_context): error_msg = "Value at %s is not a valid time delta: %s" matches = TIME_INTERVAL_RE.match(value) if not matches: raise ConfigError(error_msg % (config_context.path, value)) units = matches.group("units") if units not in TIME_INTERVAL_UNITS: raise ConfigError(error_msg % (config_context.path, value)) time_spec = {TIME_INTERVAL_UNITS[units]: int(matches.group("value"))} return datetime.timedelta(**time_spec) def valid_name_identifier(value, config_context): valid_identifier(value, config_context) if config_context.partial: return value return f"{config_context.namespace}.{value}" def build_list_of_type_validator(item_validator, allow_empty=False): """Build a validator which validates a list contains items which pass item_validator. """ def validator(value, config_context): if allow_empty and not value: return () seq = valid_list(value, config_context) if not seq: msg = "Required non-empty list at %s" raise ConfigError(msg % config_context.path) return tuple(item_validator(item, config_context) for item in seq) return validator def build_dict_name_validator(item_validator, allow_empty=False): """Build a validator which validates a list or dict, and returns a dict. Item validator must expect a "name" key, mapped to the key of the dict item""" valid = build_list_of_type_validator(item_validator, allow_empty) def validator(value, config_context): if isinstance(value, dict): value = [ { "name": name, **config, } for name, config in value.items() ] msg = "Duplicate name %%s at %s" % config_context.path name_dict = UniqueNameDict(msg) for item in valid(value, config_context): name_dict[item.name] = item return name_dict return validator def build_dict_value_validator(item_validator, allow_empty=False): """Build a validator which validates values of a dict, and returns a dict""" def validator(value, config_context): if not isinstance(value, dict): msg = "Require a dict of type %s at %s" raise ConfigError(msg % (item_validator.type_name, config_context.path)) result_dict = dict() for k, v in value.items(): result_dict[k] = item_validator(v, config_context) return result_dict return validator class ConfigContext: """An object to encapsulate the context in a configuration file. Supplied to Validators to perform validation which requires knowledge of configuration outside of the immediate configuration dictionary. """ partial = False def __init__(self, path, nodes, command_context, namespace): self.path = path self.nodes = set(nodes or []) self.command_context = command_context or {} self.namespace = namespace def build_child_context(self, path): """Construct a new ConfigContext based on this one.""" path = f"{self.path}.{path}" args = path, self.nodes, self.command_context, self.namespace return ConfigContext(*args) class PartialConfigContext: """A context object which has only a partial context. It is missing command_context and nodes. This is likely because it is being used in a named configuration fragment that does not have access to those pieces of the configuration. """ partial = True def __init__(self, path, namespace): self.path = path self.namespace = namespace def build_child_context(self, path): path = f"{self.path}.{path}" return PartialConfigContext(path, self.namespace) class NullConfigContext: path = "" nodes = set() # type: ignore command_context = {} # type: ignore namespace = MASTER_NAMESPACE partial = False @staticmethod def build_child_context(_): return NullConfigContext # TODO: extract code class Validator: """Base class for validating a collection and creating a mutable collection from the source. """ config_class: type | None = None defaults = {} # type: ignore validators = {} # type: ignore optional = False def validate(self, in_dict, config_context): if self.optional and in_dict is None: return None if in_dict is None: raise ConfigError("A %s is required." % self.type_name) shortcut_value = self.do_shortcut(in_dict) if shortcut_value: return shortcut_value config_context = self.build_context(in_dict, config_context) in_dict = self.cast(in_dict, config_context) self.validate_required_keys(in_dict) self.validate_extra_keys(in_dict) return self.build_config(in_dict, config_context) def __call__(self, in_dict, config_context=NullConfigContext): return self.validate(in_dict, config_context) @property def type_name(self): """Return a string that represents the config_class being validated. This name is used for error messages, so we strip off the word Config so the name better matches what the user sees in the config. """ return self.config_class.__name__.replace("Config", "") @property def all_keys(self): return self.config_class.required_keys + self.config_class.optional_keys def do_shortcut(self, in_dict): """Override if your validator can skip most of the validation by checking this condition. If this returns a truthy value, the validation will end immediately and return that value. """ pass def cast(self, in_dict, _): """If your validator accepts input in different formations, override this method to cast your input into a common format. """ return in_dict def build_context(self, in_dict, config_context): path = self.path_name(in_dict.get("name")) return config_context.build_child_context(path) def validate_required_keys(self, in_dict): """Check that all required keys are present.""" missing_keys = set(self.config_class.required_keys) - set(in_dict) if not missing_keys: return missing_key_str = ", ".join(missing_keys) if "name" in self.all_keys and "name" in in_dict: msg = "%s %s is missing options: %s" name = in_dict["name"] raise ConfigError(msg % (self.type_name, name, missing_key_str)) msg = "Nameless %s is missing options: %s" raise ConfigError(msg % (self.type_name, missing_key_str)) def validate_extra_keys(self, in_dict): """Check that no unexpected keys are present.""" extra_keys = set(in_dict) - set(self.all_keys) if not extra_keys: return msg = "Unknown keys in %s %s: %s" name = in_dict.get("name", "") raise ConfigError(msg % (self.type_name, name, ", ".join(extra_keys))) def set_defaults(self, output_dict, _config_context): """Set any default values for any optional values that were not specified. """ for key, value in self.defaults.items(): output_dict.setdefault(key, value) def path_name(self, name=None): return f"{self.type_name}.{name}" if name else self.type_name def post_validation(self, valid_input, config_context): """Hook to perform additional validation steps after key validation completes. """ pass def build_config(self, in_dict, config_context): """Construct the configuration by validating the contents, setting defaults, and returning an instance of the config_class. """ output_dict = self.validate_contents(in_dict, config_context) self.set_defaults(output_dict, config_context) self.post_validation(output_dict, config_context) return self.config_class(**output_dict) def validate_contents(self, input, config_context): """Override this to validate each value in the input.""" valid_input = {} for key, value in input.items(): if key in self.validators: child_context = config_context.build_child_context(key) valid_input[key] = self.validators[key](value, child_context) else: valid_input[key] = value return valid_input ================================================ FILE: tron/config/manager.py ================================================ import hashlib import logging import os from copy import deepcopy from tron import yaml from tron.config import config_parse from tron.config import ConfigError from tron.config import schema from tron.core.jobgraph import JobGraph from tron.utils import maybe_decode from tron.utils import maybe_encode log = logging.getLogger(__name__) def from_string(content): try: return yaml.safe_load(content) except yaml.yaml.error.YAMLError as e: raise ConfigError("Invalid config format: %s" % str(e)) def write(path, content): with open(path, "w") as fh: yaml.dump(content, fh) def read(path): with open(path) as fh: return from_string(fh) def write_raw(path, content): with open(path, "w") as fh: fh.write( maybe_decode(content) ) # TODO: TRON-2293 maybe_decode is a relic of Python2->Python3 migration. Remove it. def read_raw(path: str) -> str: with open(path) as fh: return fh.read() def hash_digest(content: str | bytes) -> str: return hashlib.sha1( maybe_encode(content) ).hexdigest() # TODO: TRON-2293 maybe_encode is a relic of Python2->Python3 migration. Remove it. class ManifestFile: """Manage the manifest file, which tracks name to filename.""" MANIFEST_FILENAME = "_manifest.yaml" def __init__(self, path): self.filename = os.path.join(path, self.MANIFEST_FILENAME) def create(self): if os.path.isfile(self.filename): msg = "Refusing to create manifest. File %s exists." log.info(msg % self.filename) return write(self.filename, {}) def add(self, name, filename): manifest = read(self.filename) manifest[name] = filename write(self.filename, manifest) def delete(self, name): manifest = read(self.filename) if name not in manifest: msg = "Namespace %s does not exist in manifest, cannot delete." log.info(msg % name) return del manifest[name] write(self.filename, manifest) def get_file_mapping(self): return read(self.filename) def get_file_name(self, name): return self.get_file_mapping().get(name) def __contains__(self, name): return name in self.get_file_mapping() class ConfigManager: """Read, load and write configuration.""" DEFAULT_HASH = hash_digest("") def __init__(self, config_path, manifest=None): self.config_path = config_path self.manifest = manifest or ManifestFile(config_path) self.name_mapping = None def build_file_path(self, name): name = name.replace(".", "_").replace(os.path.sep, "_") return os.path.join(self.config_path, "%s.yaml" % name) def read_raw_config(self, name: str = schema.MASTER_NAMESPACE) -> str: """Read the config file without converting to yaml.""" filename = self.manifest.get_file_name(name) return read_raw(filename) def write_config(self, name: str, content: str) -> None: loaded_content = from_string(content) self.validate_with_fragment( name, content=loaded_content, # TODO: remove this constraint after tron triggers across clusters are supported. should_validate_missing_dependency=False, ) # validate_with_fragment throws if the updated content is invalid - so if we get here # we know it's safe to reflect the update in our config store self.get_config_name_mapping()[name] = loaded_content # ...and then let's also persist the update to disk since memory is temporary, but disk is forever™ filename = self.get_filename_from_manifest(name) write_raw(filename, content) def delete_config(self, name: str) -> None: filename = self.manifest.get_file_name(name) if not filename: msg = "Namespace %s does not exist in manifest, cannot delete." log.info(msg % name) return # to avoid needing to reload from disk on every config load - we need to ensure that # we also persist config deletions into our cache self.get_config_name_mapping().pop(name, None) self.manifest.delete(name) os.remove(filename) def get_filename_from_manifest(self, name): def create_filename(): filename = self.build_file_path(name) self.manifest.add(name, filename) return filename return self.manifest.get_file_name(name) or create_filename() def validate_with_fragment( self, name, content, should_validate_missing_dependency=True, ): # NOTE: we deepcopy rather than swap values to keep this a pure function # get_config_name_mapping() returns a shared dict, so this would otherwise # actually update the mapping - which would be unwanted/need to be rolled-back # should validation fail. name_mapping = deepcopy(self.get_config_name_mapping()) name_mapping[name] = content try: JobGraph( config_parse.ConfigContainer.create(name_mapping), should_validate_missing_dependency=should_validate_missing_dependency, ) except ValueError as e: raise ConfigError(str(e)) def get_config_name_mapping(self): if self.name_mapping is None: log.info("Creating config mapping cache...") seq = self.manifest.get_file_mapping().items() self.name_mapping = {name: read(filename) for name, filename in seq} return self.name_mapping def load(self): """Return the fully constructed configuration.""" log.info("Loading full config from %s" % self.config_path) name_mapping = self.get_config_name_mapping() return config_parse.ConfigContainer.create(name_mapping) def get_hash(self, name: str) -> str: """Return a hash of the configuration contents for name.""" if name not in self: return self.DEFAULT_HASH if name in self.get_config_name_mapping(): # unfortunately, we have the parsed dict in memory. # rather than hit the disk to get the raw string - let's convert # the in-memory dict to a yaml string and hash that to save a couple # ms (in testing, ~3ms over loading from disk and ~1ms over dumping to json :p) # TODO: consider storing the hash alongside the config so that we only calculate # hashes once? return hash_digest( yaml.dump( self.get_config_name_mapping()[name], # ensure that the keys are always in a stable order sort_keys=True, ) ) # the config for any name should always be in our name mapping # ...but just in case, let's fallback to reading from disk. log.warning("%s not found in name mapping - falling back to hashing contents on disk!") return hash_digest(self.read_raw_config(name)) def __contains__(self, name): return name in self.manifest def get_namespaces(self) -> list[str]: return list(self.manifest.get_file_mapping().keys()) def create_new_config(path, master_content): """Create a new configuration directory with master config.""" os.makedirs(path) manager = ConfigManager(path) manager.manifest.create() filename = manager.get_filename_from_manifest(schema.MASTER_NAMESPACE) write_raw(filename, master_content) ================================================ FILE: tron/config/schedule_parse.py ================================================ """ Parse and validate scheduler configuration and return immutable structures. """ import calendar import datetime import re from collections import namedtuple from tron.config import config_utils from tron.config import ConfigError from tron.config import schema from tron.utils import crontab ConfigGenericSchedule = schema.config_object_factory( "ConfigGenericSchedule", ["type", "value"], ["jitter"], ) ConfigGrocScheduler = namedtuple( "ConfigGrocScheduler", "original ordinals weekdays monthdays months timestr jitter", ) ConfigCronScheduler = namedtuple( "ConfigCronScheduler", "original minutes hours monthdays months weekdays ordinals jitter", ) ConfigDailyScheduler = namedtuple( "ConfigDailyScheduler", "original hour minute second days jitter", ) class ScheduleParseError(ConfigError): pass def pad_sequence(seq, size, padding=None): """Force a sequence to size. Pad with padding if too short, and ignore extra pieces if too long.""" return (list(seq) + [padding for _ in range(size)])[:size] def schedule_config_from_string(schedule, config_context): """Return a scheduler config object from a string.""" schedule = schedule.strip() name, schedule_config = pad_sequence( schedule.split(None, 1), 2, padding="", ) if name not in schedulers: config = ConfigGenericSchedule("groc daily", schedule, jitter=None) return parse_groc_expression(config, config_context) config = ConfigGenericSchedule(name, schedule_config, jitter=None) return validate_generic_schedule_config(config, config_context) def validate_generic_schedule_config(config, config_context): return schedulers[config.type](config, config_context) # TODO: remove in 0.7 def schedule_config_from_legacy_dict(schedule, config_context): """Support old style schedules as dicts.""" if "start_time" in schedule or "days" in schedule: start_time = schedule.get("start_time", "00:00:00") days = schedule.get("days", "") scheduler_config = f"{start_time} {days}" config = ConfigGenericSchedule("daily", scheduler_config, None) return valid_daily_scheduler(config, config_context) path = config_context.path raise ConfigError(f"Unknown scheduler at {path}: {schedule}") def valid_schedule(schedule, config_context): if isinstance(schedule, str): return schedule_config_from_string(schedule, config_context) if "type" not in schedule: return schedule_config_from_legacy_dict(schedule, config_context) schedule = ScheduleValidator().validate(schedule, config_context) return validate_generic_schedule_config(schedule, config_context) def valid_daily_scheduler(config, config_context): """Daily scheduler, accepts a time of day and an optional list of days.""" schedule_config = config.value time_string, days = pad_sequence(schedule_config.split(), 2) time_string = time_string or "00:00:00" time_spec = config_utils.valid_time(time_string, config_context) days = config_utils.valid_string(days or "", config_context) def valid_day(day): if day not in CONVERT_DAYS_INT: raise ConfigError( f"Unknown day {day} at {config_context.path}", ) return CONVERT_DAYS_INT[day] original = f"{time_string} {days}" weekdays = {valid_day(day) for day in days or ()} return ConfigDailyScheduler( original, time_spec.hour, time_spec.minute, time_spec.second, weekdays, jitter=config.jitter, ) def normalize_weekdays(seq): return seq[6:7] + seq[:6] def day_canonicalization_map(): """Build a map of weekday synonym to int index 0-6 inclusive.""" canon_map = dict() # 7-element lists with weekday names in order weekday_lists = [ normalize_weekdays(calendar.day_name), normalize_weekdays(calendar.day_abbr), ( "u", "m", "t", "w", "r", "f", "s", ), ( "su", "mo", "tu", "we", "th", "fr", "sa", ), ] for day_list in weekday_lists: for day_name_synonym, day_index in zip(day_list, range(7)): canon_map[day_name_synonym] = day_index canon_map[day_name_synonym.lower()] = day_index canon_map[day_name_synonym.upper()] = day_index return canon_map # Canonicalize weekday names to integer indices CONVERT_DAYS_INT = day_canonicalization_map() # day name/abbrev => {0123456} def month_canonicalization_map(): """Build a map of month synonym to int index 0-11 inclusive.""" canon_map = dict() # calendar stores month data with a useless element in front. cut it off. monthname_lists = (calendar.month_name[1:], calendar.month_abbr[1:]) for month_list in monthname_lists: for key, value in zip(month_list, range(1, 13)): canon_map[key] = value canon_map[key.lower()] = value return canon_map # Canonicalize month names to integer indices # month name/abbrev => {0 <= k <= 11} CONVERT_MONTHS = month_canonicalization_map() def build_groc_schedule_parser_re(): """Build a regular expression that matches this: ("every"|ordinal) (day) ["of|in" (monthspec)] (["at"] HH:MM) ordinal - comma-separated list of "1st" and so forth days - comma-separated list of days of the week (for example, "mon", "tuesday", with both short and long forms being accepted); "every day" is equivalent to "every mon,tue,wed,thu,fri,sat,sun" monthspec - comma-separated list of month names (for example, "jan", "march", "sep"). If omitted, implies every month. You can also say "month" to mean every month, as in "1,8th,15,22nd of month 09:00". HH:MM - time of day in 24 hour time. This is a slightly more permissive version of Google App Engine's schedule parser, documented here: http://code.google.com/appengine/docs/python/config/cron.html#The_Schedule_Format """ # m|mon|monday|...|day DAY_VALUES = "|".join(list(CONVERT_DAYS_INT.keys()) + ["day"]) # jan|january|...|month MONTH_VALUES = "|".join(list(CONVERT_MONTHS.keys()) + ["month"]) DATE_SUFFIXES = "st|nd|rd|th" # every|1st|2nd|3rd (also would accept 3nd, 1rd, 4st) MONTH_DAYS_EXPR = r"(?Pevery|((\d+(%s),?)+))?" % DATE_SUFFIXES DAYS_EXPR = r"((?P((%s),?)+))?" % DAY_VALUES MONTHS_EXPR = r"((in|of)\s+(?P((%s),?)+))?" % MONTH_VALUES # [at] 00:00 TIME_EXPR = r"((at\s+)?(?P